Merge remote-tracking branch 'origin/main' into sframe

Author: labath

.ci/compute_projects_test.py

@@ -1,7 +1,7 @@
 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-"""Does some stuff."""
+"""Tests for compute_projects.py"""
 
 import unittest
 

.ci/metrics/metrics.py

@@ -1,3 +1,13 @@
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""Collects Github metrics and uploads them to Grafana.
+
+This script contains machinery that will pull metrics periodically from Github
+about workflow runs. It will upload the collected metrics to the specified
+Grafana instance.
+"""
+
 import collections
 import datetime
 import github

.github/workflows/premerge.yaml

@@ -34,11 +34,6 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 2
-      - name: Setup ccache
-        uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17
-        with:
-          variant: "sccache"
-          max-size: "2000M"
       - name: Build and Test
         # Mark the job as a success even if the step fails so that people do
         # not get notified while the new premerge pipeline is in an
@@ -62,6 +57,13 @@ jobs:
           export CC=/opt/llvm/bin/clang
           export CXX=/opt/llvm/bin/clang++
 
+          # This environment variable is passes into the container through the
+          # runner pod definition. This differs between our two clusters which
+          # why we do not hardcode it.
+          export SCCACHE_GCS_BUCKET=$CACHE_GCS_BUCKET
+          export SCCACHE_GCS_RW_MODE=READ_WRITE
+          sccache --start-server
+
           ./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}" "${runtimes_check_targets_needs_reconfig}" "${enable_cir}"
       - name: Upload Artifacts
         if: '!cancelled()'
@@ -86,11 +88,6 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: 2
-      - name: Setup ccache
-        uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17
-        with:
-          variant: "sccache"
-          max-size: "2000M"
       - name: Compute Projects
         id: vars
         run: |
@@ -113,7 +110,7 @@ jobs:
         shell: cmd
         run: |
           call C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64
-          bash .ci/monolithic-windows.sh "${{ steps.vars.outputs.windows-projects }}" "${{ steps.vars.outputs.windows-check-targets }}"
+          bash -c "export SCCACHE_GCS_BUCKET=$CACHE_GCS_BUCKET; export SCCACHE_GCS_RW_MODE=READ_WRITE; sccache --start-server; .ci/monolithic-windows.sh \"${{ steps.vars.outputs.windows-projects }}\" \"${{ steps.vars.outputs.windows-check-targets }}\""
       - name: Upload Artifacts
         if: '!cancelled()'
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0

bolt/utils/nfc-check-setup.py

@@ -7,6 +7,8 @@
 import sys
 import textwrap
 
+msg_prefix = "\n> NFC-Mode:"
+
 def get_relevant_bolt_changes(dir: str) -> str:
     # Return a list of bolt source changes that are relevant to testing.
     all_changes = subprocess.run(
@@ -42,14 +44,32 @@ def get_git_ref_or_rev(dir: str) -> str:
     cmd_rev = "git rev-parse --short HEAD"
     return subprocess.check_output(shlex.split(cmd_rev), cwd=dir, text=True).strip()
 
+def switch_back(
+    switch_back: bool, stash: bool, source_dir: str, old_ref: str, new_ref: str
+):
+    # Switch back to the current revision if needed and inform the user of where
+    # the HEAD is. Must be called after checking out the previous commit on all
+    # exit paths.
+    if switch_back:
+        print(f"{msg_prefix} Switching back to current revision..")
+        if stash:
+            subprocess.run(shlex.split("git stash pop"), cwd=source_dir)
+        subprocess.run(shlex.split(f"git checkout {old_ref}"), cwd=source_dir)
+    else:
+        print(
+            f"The repository {source_dir} has been switched from {old_ref} "
+            f"to {new_ref}. Local changes were stashed. Switch back using\n\t"
+            f"git checkout {old_ref}\n"
+        )
 
 def main():
     parser = argparse.ArgumentParser(
         description=textwrap.dedent(
             """
-            This script builds two versions of BOLT (with the current and
-            previous revision) and sets up symlink for llvm-bolt-wrapper.
-            Passes the options through to llvm-bolt-wrapper.
+            This script builds two versions of BOLT:
+            llvm-bolt.new, using the current revision, and llvm-bolt.old using
+            the previous revision. These can be used to check whether the
+            current revision changes BOLT's functional behavior.
             """
         )
     )
@@ -59,6 +79,12 @@ def main():
         default=os.getcwd(),
         help="Path to BOLT build directory, default is current " "directory",
     )
+    parser.add_argument(
+        "--create-wrapper",
+        default=False,
+        action="store_true",
+        help="Sets up llvm-bolt as a symlink to llvm-bolt-wrapper. Passes the options through to llvm-bolt-wrapper.",
+    )
     parser.add_argument(
         "--check-bolt-sources",
         default=False,
@@ -76,28 +102,42 @@ def main():
         default="HEAD^",
         help="Revision to checkout to compare vs HEAD",
     )
+
+    # When creating a wrapper, pass any unknown arguments to it. Otherwise, die.
     args, wrapper_args = parser.parse_known_args()
-    bolt_path = f"{args.build_dir}/bin/llvm-bolt"
+    if not args.create_wrapper and len(wrapper_args) > 0:
+        parser.parse_args()
 
+    # Find the repo directory.
     source_dir = None
-    # find the repo directory
-    with open(f"{args.build_dir}/CMakeCache.txt") as f:
-        for line in f:
-            m = re.match(r"LLVM_SOURCE_DIR:STATIC=(.*)", line)
-            if m:
-                source_dir = m.groups()[0]
-    if not source_dir:
-        sys.exit("Source directory is not found")
-
-    script_dir = os.path.dirname(os.path.abspath(__file__))
-    wrapper_path = f"{script_dir}/llvm-bolt-wrapper.py"
-    # build the current commit
+    try:
+        CMCacheFilename = f"{args.build_dir}/CMakeCache.txt"
+        with open(CMCacheFilename) as f:
+            for line in f:
+                m = re.match(r"LLVM_SOURCE_DIR:STATIC=(.*)", line)
+                if m:
+                    source_dir = m.groups()[0]
+        if not source_dir:
+            raise Exception(f"Source directory not found: '{CMCacheFilename}'")
+    except Exception as e:
+        sys.exit(e)
+
+    # Clean the previous llvm-bolt if it exists.
+    bolt_path = f"{args.build_dir}/bin/llvm-bolt"
+    if os.path.exists(bolt_path):
+        os.remove(bolt_path)
+
+    # Build the current commit.
+    print(f"{msg_prefix} Building current revision..")
     subprocess.run(
         shlex.split("cmake --build . --target llvm-bolt"), cwd=args.build_dir
     )
-    # rename llvm-bolt
+
+    if not os.path.exists(bolt_path):
+        sys.exit(f"Failed to build the current revision: '{bolt_path}'")
+
+    # Rename llvm-bolt and memorize the old hash for logging.
     os.replace(bolt_path, f"{bolt_path}.new")
-    # memorize the old hash for logging
     old_ref = get_git_ref_or_rev(source_dir)
 
     if args.check_bolt_sources:
@@ -110,7 +150,7 @@ def main():
             print(f"BOLT source changes were found:\n{file_changes}")
             open(marker, "a").close()
 
-    # determine whether a stash is needed
+    # Determine whether a stash is needed.
     stash = subprocess.run(
         shlex.split("git status --porcelain"),
         cwd=source_dir,
@@ -119,42 +159,59 @@ def main():
         text=True,
     ).stdout
     if stash:
-        # save local changes before checkout
+        # Save local changes before checkout.
         subprocess.run(shlex.split("git stash push -u"), cwd=source_dir)
-    # check out the previous/cmp commit
+
+    # Check out the previous/cmp commit and get its commit hash for logging.
     subprocess.run(shlex.split(f"git checkout -f {args.cmp_rev}"), cwd=source_dir)
-    # get the parent commit hash for logging
     new_ref = get_git_ref_or_rev(source_dir)
-    # build the previous commit
+
+    # Build the previous commit.
+    print(f"{msg_prefix} Building previous revision..")
     subprocess.run(
         shlex.split("cmake --build . --target llvm-bolt"), cwd=args.build_dir
     )
-    # rename llvm-bolt
+
+    # Rename llvm-bolt.
+    if not os.path.exists(bolt_path):
+        print(f"Failed to build the previous revision: '{bolt_path}'")
+        switch_back(args.switch_back, stash, source_dir, old_ref, new_ref)
+        sys.exit(1)
     os.replace(bolt_path, f"{bolt_path}.old")
-    # set up llvm-bolt-wrapper.ini
-    ini = subprocess.check_output(
-        shlex.split(f"{wrapper_path} {bolt_path}.old {bolt_path}.new") + wrapper_args,
-        text=True,
+
+    # Symlink llvm-bolt-wrapper
+    if args.create_wrapper:
+        print(f"{msg_prefix} Creating llvm-bolt wrapper..")
+        script_dir = os.path.dirname(os.path.abspath(__file__))
+        wrapper_path = f"{script_dir}/llvm-bolt-wrapper.py"
+        try:
+            # Set up llvm-bolt-wrapper.ini
+            ini = subprocess.check_output(
+                shlex.split(f"{wrapper_path} {bolt_path}.old {bolt_path}.new")
+                + wrapper_args,
+                text=True,
+            )
+            with open(f"{args.build_dir}/bin/llvm-bolt-wrapper.ini", "w") as f:
+                f.write(ini)
+            os.symlink(wrapper_path, bolt_path)
+        except Exception as e:
+            print("Failed to create a wrapper:\n" + str(e))
+            switch_back(args.switch_back, stash, source_dir, old_ref, new_ref)
+            sys.exit(1)
+
+    switch_back(args.switch_back, stash, source_dir, old_ref, new_ref)
+
+    print(
+        f"{msg_prefix} Completed!\nBuild directory {args.build_dir} is ready for"
+        " NFC-Mode comparison between the two revisions."
     )
-    with open(f"{args.build_dir}/bin/llvm-bolt-wrapper.ini", "w") as f:
-        f.write(ini)
-    # symlink llvm-bolt-wrapper
-    os.symlink(wrapper_path, bolt_path)
-    if args.switch_back:
-        if stash:
-            subprocess.run(shlex.split("git stash pop"), cwd=source_dir)
-        subprocess.run(shlex.split(f"git checkout {old_ref}"), cwd=source_dir)
-    else:
+
+    if args.create_wrapper:
         print(
-            f"The repository {source_dir} has been switched from {old_ref} "
-            f"to {new_ref}. Local changes were stashed. Switch back using\n\t"
-            f"git checkout {old_ref}\n"
+            "Can run BOLT tests using:\n"
+            "\tbin/llvm-lit -sv tools/bolt/test\nor\n"
+            "\tbin/llvm-lit -sv tools/bolttests"
         )
-    print(
-        f"Build directory {args.build_dir} is ready to run BOLT tests, e.g.\n"
-        "\tbin/llvm-lit -sv tools/bolt/test\nor\n"
-        "\tbin/llvm-lit -sv tools/bolttests"
-    )
 
 
 if __name__ == "__main__":

clang-tools-extra/clang-doc/BitcodeReader.cpp

@@ -384,6 +384,8 @@ static llvm::Error parseRecord(const Record &R, unsigned ID,
     return decodeRecord(R, I->Path, Blob);
   case REFERENCE_FIELD:
     return decodeRecord(R, F, Blob);
+  case REFERENCE_FILE:
+    return decodeRecord(R, I->DocumentationFileName, Blob);
   default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "invalid field for Reference");

clang-tools-extra/clang-doc/BitcodeWriter.cpp

@@ -210,6 +210,7 @@ static const llvm::IndexedMap<RecordIdDsc, RecordIdToIndexFunctor>
           {REFERENCE_TYPE, {"RefType", &genIntAbbrev}},
           {REFERENCE_PATH, {"Path", &genStringAbbrev}},
           {REFERENCE_FIELD, {"Field", &genIntAbbrev}},
+          {REFERENCE_FILE, {"File", &genStringAbbrev}},
           {TEMPLATE_PARAM_CONTENTS, {"Contents", &genStringAbbrev}},
           {TEMPLATE_SPECIALIZATION_OF,
            {"SpecializationOf", &genSymbolIdAbbrev}},
@@ -286,7 +287,7 @@ static const std::vector<std::pair<BlockId, std::vector<RecordId>>>
         // Reference Block
         {BI_REFERENCE_BLOCK_ID,
          {REFERENCE_USR, REFERENCE_NAME, REFERENCE_QUAL_NAME, REFERENCE_TYPE,
-          REFERENCE_PATH, REFERENCE_FIELD}},
+          REFERENCE_PATH, REFERENCE_FIELD, REFERENCE_FILE}},
         // Template Blocks.
         {BI_TEMPLATE_BLOCK_ID, {}},
         {BI_TEMPLATE_PARAM_BLOCK_ID, {TEMPLATE_PARAM_CONTENTS}},
@@ -479,6 +480,7 @@ void ClangDocBitcodeWriter::emitBlock(const Reference &R, FieldId Field) {
   emitRecord((unsigned)R.RefType, REFERENCE_TYPE);
   emitRecord(R.Path, REFERENCE_PATH);
   emitRecord((unsigned)Field, REFERENCE_FIELD);
+  emitRecord(R.DocumentationFileName, REFERENCE_FILE);
 }
 
 void ClangDocBitcodeWriter::emitBlock(const FriendInfo &R) {

clang-tools-extra/clang-doc/BitcodeWriter.h

@@ -140,6 +140,7 @@ enum RecordId {
   REFERENCE_TYPE,
   REFERENCE_PATH,
   REFERENCE_FIELD,
+  REFERENCE_FILE,
   TEMPLATE_PARAM_CONTENTS,
   TEMPLATE_SPECIALIZATION_OF,
   TYPEDEF_USR,