Documentation on reproducing BUCK CI failures

Author: mergennachin

.ci/docker/common/install_buck.sh

@@ -8,18 +8,23 @@
 set -ex
 
 install_ubuntu() {
-  apt-get update
-  apt-get install -y zstd
+  pip install zstd certifi
+  SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+  REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
+  
+  # Create a cache directory for buck2
+  CACHE_DIR="/tmp/buck2_cache"
+  mkdir -p "${CACHE_DIR}"
 
-  BUCK2=buck2-x86_64-unknown-linux-gnu.zst
-  wget -q "https://github.com/facebook/buck2/releases/download/${BUCK2_VERSION}/${BUCK2}"
-  zstd -d "${BUCK2}" -o buck2
+  # Run resolve_buck.py to get the buck2 binary
+  BUCK2_PATH=$(python "${REPO_ROOT}/tools/cmake/resolve_buck.py" --cache_dir "${CACHE_DIR}")
 
-  chmod +x buck2
-  mv buck2 /usr/bin/
+  # Move buck2 to /usr/bin/
+  mv "${BUCK2_PATH}" /usr/bin/buck2
+  chmod +x /usr/bin/buck2
 
-  rm "${BUCK2}"
-  # Cleanup package manager
+  # Cleanup
+  rm -rf "${CACHE_DIR}"
   apt-get autoclean && apt-get clean
   rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 }

.ci/scripts/setup-macos.sh

@@ -13,36 +13,21 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
 read -r BUILD_TOOL BUILD_MODE EDITABLE < <(parse_args "$@")
 
 install_buck() {
-  if ! command -v zstd &> /dev/null; then
-    brew install zstd
-  fi
+  pip install zstd certifi
+  # Use resolve_buck.py to download and install buck2
+  # The script handles platform detection and version management
+  REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 
-  if ! command -v curl &> /dev/null; then
-    brew install curl
-  fi
+  # Create a cache directory for buck2
+  CACHE_DIR="${HOME}/.cache/buck2"
+  mkdir -p "${CACHE_DIR}"
+
+  # Run resolve_buck.py to get the buck2 binary
+  BUCK2_PATH=$(python "${REPO_ROOT}/tools/cmake/resolve_buck.py" --cache_dir "${CACHE_DIR}")
 
-  pushd .ci/docker
-  # TODO(huydo): This is a one-off copy of buck2 2024-05-15 to unblock Jon and
-  # re-enable ShipIt. It’s not ideal that upgrading buck2 will require a manual
-  # update the cached binary on S3 bucket too. Let me figure out if there is a
-  # way to correctly implement the previous setup of installing a new version of
-  # buck2 only when it’s needed. AFAIK, the complicated part was that buck2
-  # --version doesn't say anything w.r.t its release version, i.e. 2024-05-15.
-  # See D53878006 for more details.
-  #
-  # If you need to upgrade buck2 version on S3, please reach out to Dev Infra
-  # team for help.
-  BUCK2_VERSION=$(cat ci_commit_pins/buck2.txt)
-  BUCK2=buck2-aarch64-apple-darwin-${BUCK2_VERSION}.zst
-  curl -s "https://ossci-macos.s3.amazonaws.com/${BUCK2}" -o "${BUCK2}"
-
-  zstd -d "${BUCK2}" -o buck2
-
-  chmod +x buck2
-  mv buck2 /opt/homebrew/bin
-
-  rm "${BUCK2}"
-  popd
+  # Move buck2 to /opt/homebrew/bin
+  sudo mv "${BUCK2_PATH}" /opt/homebrew/bin/buck2
+  sudo chmod +x /opt/homebrew/bin/buck2
 
   # Kill all running buck2 daemon for a fresh start
   buck2 killall || true
@@ -114,8 +99,8 @@ setup_macos_env_variables() {
 setup_macos_env_variables
 # NB: we need buck2 in all cases because cmake build also depends on calling
 # buck2 atm
-install_buck
 brew install libomp
+install_buck
 install_pip_dependencies
 
 # TODO(huydhn): Unlike our self-hosted runner, GitHub runner doesn't have access

CONTRIBUTING.md

@@ -329,6 +329,23 @@ the CI (continuous integration) jobs. If `main` is broken, consider rebasing
 your PR onto the `viable/strict` branch, which points to the most recent
 all-green commit.
 
+### BUCK tests
+In addition to CMake, we run `buck` tests in our CI pipeline to ensure continued integration with certain customers. We recognize that `buck` is not a widely used build system and may not be installed on your local machine. If you encounter a failure in the `buck` CI and wish to reproduce it locally, follow these steps:
+
+```
+# Install buck
+BUCK2_PATH=$(python tools/cmake/resolve_buck.py --cache_dir=/tmp/)
+mv "${BUCK2_PATH}" ~/buck2
+chmod +x ~/buck2
+
+# Build example
+~/buck2 build //runtime/platform/... # replace with other targets
+
+# Test example
+~/buck2 test //runtime/platform/...  # replace with other targets
+```
+
+Please refer to the [official buck documentation](https://buck.build/command/build.html).
  
 
 ## Updating Documentation

pyproject.toml

@@ -5,8 +5,6 @@ requires = [
   "pyyaml",  # Imported by the kernel codegen tools.
   "setuptools>=63",  # For building the pip package contents.
   "wheel",  # For building the pip package archive.
-  "zstd",  # Imported by resolve_buck.py.
-  "certifi",  # Imported by resolve_buck.py.
 ]
 build-backend = "setuptools.build_meta"
 

requirements-dev.txt

@@ -5,7 +5,5 @@ pip>=23  # For building the pip package.
 pyyaml  # Imported by the kernel codegen tools.
 setuptools>=63  # For building the pip package contents.
 wheel  # For building the pip package archive.
-zstd  # Imported by resolve_buck.py.
-certifi  # Imported by resolve_buck.py.
 lintrunner==0.12.7
 lintrunner-adapters==0.12.6