Merge branch 'llvm:main' into clang-tools-headers

Author: jathu

.ci/generate-buildkite-pipeline-premerge

@@ -9,7 +9,7 @@
 
 #
 # This file generates a Buildkite pipeline that triggers the various CI jobs for
-# the LLVM project during pre-commit CI (each time a Phabricator diff is uploaded).
+# the LLVM project during pre-commit CI.
 #
 # See https://buildkite.com/docs/agent/v3/cli-pipeline#pipeline-format.
 #
@@ -22,19 +22,18 @@ set -o pipefail
 
 # Environment variables script works with:
 
-# Fetch origin/main to have an up to date merge base for main...HEAD diff.
-git fetch origin main:main
+# Set by buildkite
+: ${BUILDKITE_PULL_REQUEST_BASE_BRANCH:=}
+: ${BUILDKITE_COMMIT:=}
+: ${BUILDKITE_BRANCH:=}
+# Fetch origin to have an up to date merge base for the diff.
+git fetch origin
 # List of files affected by this commit
-: ${MODIFIED_FILES:=$(git diff --name-only main...HEAD)}
+: ${MODIFIED_FILES:=$(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD)}
 # Filter rules for generic windows tests
 : ${WINDOWS_AGENTS:='{"queue": "windows"}'}
 # Filter rules for generic linux tests
 : ${LINUX_AGENTS:='{"queue": "linux"}'}
-# Service agents, for interacting with Phabricator.
-: ${SERVICE_AGENTS:='{"queue": "service"}'}
-# Set by buildkite
-: ${BUILDKITE_COMMIT:=}
-: ${BUILDKITE_BRANCH:=}
 
 reviewID="$(git log --format=%B -n 1 | sed -nE 's/^Review-ID:[[:space:]]*(.+)$/\1/p')"
 if [[ "${reviewID}" != "" ]]; then
@@ -284,28 +283,3 @@ if [[ "${windows_projects}" != "" ]]; then
   - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'
 EOF
 fi
-
-# If build was triggered from a Phabricator review - send an update back.
-if [[ -n "${ph_target_phid:-}" ]]; then
-  cat << EOF
-- continue_on_failure: true
-  wait: '~'
-- label: ':phabricator: update build status on Phabricator'
-  agents: ${SERVICE_AGENTS}
-  artifact_paths:
-  - 'artifacts/**/*'
-  commands:
-  - export SRC=\$\${BUILDKITE_BUILD_PATH}/llvm-premerge-checks
-  - rm -rf \$\${SRC}
-  - git clone --depth 1 https://github.com/google/llvm-premerge-checks.git "\$\${SRC}"
-  - cd \$\${SRC}
-  - git fetch origin "main":x
-  - git checkout x
-  - echo "llvm-premerge-checks commit"
-  - git rev-parse HEAD
-  - pip install -q -r \$\${SRC}/scripts/requirements.txt
-  - cd "\$\$BUILDKITE_BUILD_CHECKOUT_PATH"
-  - \$\${SRC}/scripts/summary.py
-  timeout_in_minutes: 10
-EOF
-fi

.ci/generate-buildkite-pipeline-scheduled

@@ -33,6 +33,9 @@ jobs:
   backport-commits:
     name: Backport Commits
     runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
     if: >-
       (github.repository == 'llvm/llvm-project') &&
       !startswith(github.event.comment.body, '<!--IGNORE-->') &&
@@ -58,37 +61,8 @@ jobs:
           printf "%s" "$COMMENT_BODY" |
           ./llvm/utils/git/github-automation.py \
           --repo "$GITHUB_REPOSITORY" \
-          --token ${{ secrets.RELEASE_WORKFLOW_PUSH_SECRET }} \
+          --token "${{ secrets.RELEASE_WORKFLOW_PR_CREATE }}" \
           release-workflow \
+          --branch-repo-token ${{ secrets.RELEASE_WORKFLOW_PUSH_SECRET }} \
           --issue-number ${{ github.event.issue.number }} \
-          --phab-token ${{ secrets.RELEASE_WORKFLOW_PHAB_TOKEN }} \
-          auto
-
-  create-pull-request:
-    name: Create Pull Request
-    runs-on: ubuntu-latest
-    if: >-
-      (github.repository == 'llvm/llvm-project') &&
-      !startswith(github.event.comment.body, '<!--IGNORE-->') &&
-      contains(github.event.comment.body, '/branch ')
-
-    steps:
-      - name: Fetch LLVM sources
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-
-      - name: Setup Environment
-        run: |
-          pip install -r ./llvm/utils/git/requirements.txt
-
-      - name: Create Pull Request
-        run: |
-          printf "%s" "$COMMENT_BODY" |
-          ./llvm/utils/git/github-automation.py \
-          --repo "$GITHUB_REPOSITORY" \
-          --token ${{ secrets.RELEASE_WORKFLOW_PUSH_SECRET }} \
-          release-workflow \
-          --issue-number ${{ github.event.issue.number }} \
-          --phab-token ${{ secrets.RELEASE_WORKFLOW_PHAB_TOKEN }} \
           auto

.github/workflows/issue-release-workflow.yml

@@ -6,6 +6,8 @@ permissions:
 on:
   workflow_dispatch:
   push:
+    branches:
+      - 'main'
     paths:
       - 'clang/bindings/python/**'
       - 'clang/tools/libclang/**'
@@ -30,6 +32,7 @@ jobs:
   check-clang-python:
     # Build libclang and then run the libclang Python binding's unit tests.
     name: Build and run Python unit tests
+    if: github.repository == 'llvm/llvm-project'
     strategy:
       fail-fast: false
       matrix:

.github/workflows/libclang-python-tests.yml

@@ -23,8 +23,8 @@ on:
       - 'cmake/**'
       - '.github/workflows/libcxx-build-and-test.yaml'
   schedule:
-    # Run nightly at 8 AM UTC (or roughly 3 AM eastern)
-    - cron: '0 3 * * *'
+    # Run nightly at 08:00 UTC (aka 00:00 Pacific, aka 03:00 Eastern)
+    - cron: '0 8 * * *'
 
 permissions:
   contents: read # Default everything to read-only
@@ -199,3 +199,45 @@ jobs:
             **/CMakeError.log
             **/CMakeOutput.log
             **/crash_diagnostics/*
+  windows:
+    runs-on: windows-2022
+    needs: [ stage1 ]
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - { config: clang-cl-dll, mingw: false }
+        - { config: clang-cl-static, mingw: false }
+        - { config: clang-cl-no-vcruntime, mingw: false }
+        - { config: clang-cl-debug, mingw: false }
+        - { config: clang-cl-static-crt, mingw: false }
+        - { config: mingw-dll, mingw: true }
+        - { config: mingw-static, mingw: true }
+        - { config: mingw-dll-i686, mingw: true }
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install dependencies
+        run: |
+          choco install -y ninja wget
+          pip install psutil
+      - name: Install a current LLVM
+        if: ${{ matrix.mingw != true }}
+        run: |
+          choco install -y llvm --version=17.0.6
+      - name: Install llvm-mingw
+        if: ${{ matrix.mingw == true }}
+        run: |
+          curl -LO https://github.com/mstorsjo/llvm-mingw/releases/download/20231128/llvm-mingw-20231128-ucrt-x86_64.zip
+          powershell Expand-Archive llvm-mingw*.zip -DestinationPath .
+          del llvm-mingw*.zip
+          mv llvm-mingw* c:\llvm-mingw
+          echo "c:\llvm-mingw\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append
+      - name: Add Git Bash to the path
+        run: |
+          echo "c:\Program Files\Git\usr\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append
+      - name: Set up the MSVC dev environment
+        if: ${{ matrix.mingw != true }}
+        uses: ilammy/msvc-dev-cmd@v1
+      - name: Build and test
+        run: |
+          bash libcxx/utils/ci/run-buildbot ${{ matrix.config }}

.github/workflows/libcxx-build-and-test.yaml

@@ -14,7 +14,7 @@ on:
         required: false
       os_list:
         required: false
-        default: '["ubuntu-latest", "windows-2019", "macOS-11"]'
+        default: '["ubuntu-latest", "windows-2019", "macOS-13"]'
       python_version:
         required: false
         type: string
@@ -38,9 +38,7 @@ on:
         type: string
         # Use windows-2019 due to:
         # https://developercommunity.visualstudio.com/t/Prev-Issue---with-__assume-isnan-/1597317
-        # We're using a specific version of macOS due to:
-        # https://github.com/actions/virtual-environments/issues/5900
-        default: '["ubuntu-latest", "windows-2019", "macOS-11"]'
+        default: '["ubuntu-latest", "windows-2019", "macOS-13"]'
 
       python_version:
         required: false
@@ -95,7 +93,8 @@ jobs:
           # run creates a new cache entry so we want to ensure that we have
           # enough cache space for all the tests to run at once and still
           # fit under the 10 GB limit.
-          max-size: 500M
+          # Default to 2G to workaround: https://github.com/hendrikmuhs/ccache-action/issues/174
+          max-size: 2G
           key: ${{ matrix.os }}
           variant: sccache
       - name: Build and Test

.github/workflows/llvm-project-tests.yml

@@ -27,31 +27,13 @@ concurrency:
   cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
 
 jobs:
-  check_all:
+  check-all:
     if: github.repository_owner == 'llvm'
-    name: Test llvm,clang,libclc
+    name: Build and Test
     uses: ./.github/workflows/llvm-project-tests.yml
     with:
       build_target: check-all
-      projects: clang;libclc
-
-  # These need to be separate from the check_all job, becuase there is not enough disk
-  # space to build all these projects on Windows.
-  build_lldb:
-    if: github.repository_owner == 'llvm'
-    name: Build lldb
-    uses: ./.github/workflows/llvm-project-tests.yml
-    with:
-      build_target: ''
-      projects: clang;lldb
-
-  check_lld:
-    if: github.repository_owner == 'llvm'
-    name: Test lld
-    uses: ./.github/workflows/llvm-project-tests.yml
-    with:
-      build_target: check-lld
-      projects: lld
+      projects: clang;lld;libclc;lldb
 
   abi-dump-setup:
     if: github.repository_owner == 'llvm'

.github/workflows/llvm-tests.yml

@@ -16,7 +16,7 @@ def get_version_from_tag(tag):
 
     m = re.match("llvmorg-([0-9]+)-init", tag)
     if m:
-        return (m.group(1), "0", "0")
+        return (m.group(1), "1", "0")
 
     raise Exception(f"error: Tag is not valid: {tag}")
 

.github/workflows/version-check.py

@@ -61,7 +61,7 @@ Cold functions table header
 ```
 
 ### Functions table
-Hot and cold functions tables share the encoding except difference marked below.
+Hot and cold functions tables share the encoding except differences marked below.
 Header:
 | Entry  | Encoding | Description |
 | ------ | ----- | ----------- |
@@ -80,18 +80,23 @@ Hot indices are delta encoded, implicitly starting at zero.
 | `Address` | Continuous, Delta, ULEB128 | Function address in the output binary |
 | `HotIndex` | Delta, ULEB128 | Cold functions only: index of corresponding hot function in hot functions table |
 | `NumEntries` | ULEB128 | Number of address translation entries for a function |
+| `EqualElems` | ULEB128 | Hot functions only: number of equal offsets in the beginning of a function |
+| `BranchEntries` | Bitmask, `alignTo(EqualElems, 8)` bits | Hot functions only: if `EqualElems` is non-zero, bitmask denoting entries with `BRANCHENTRY` bit |
 
-Function header is followed by `NumEntries` pairs of offsets for current
-function.
+Function header is followed by `EqualElems` offsets (hot functions only) and
+`NumEntries-EqualElems` (`NumEntries` for cold functions) pairs of offsets for
+current function.
 
 ### Address translation table
 Delta encoding means that only the difference with the previous corresponding
 entry is encoded. Input offsets implicitly start at zero.
 | Entry  | Encoding | Description |
 | ------ | ------| ----------- |
 | `OutputOffset` | Continuous, Delta, ULEB128 | Function offset in output binary |
-| `InputOffset` | Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit |
+| `InputOffset` | Optional, Delta, SLEB128 | Function offset in input binary with `BRANCHENTRY` LSB bit |
 
 `BRANCHENTRY` bit denotes whether a given offset pair is a control flow source
 (branch or call instruction). If not set, it signifies a control flow target
 (basic block offset).
+`InputAddr` is omitted for equal offsets in input and output function. In this
+case, `BRANCHENTRY` bits are encoded separately in a `BranchEntries` bitvector.