convert benchmarks to litCI

.lightning/workflows/benchmark.yml

@@ -0,0 +1,82 @@
+trigger:
+  push:
+    branches: ["master", "release/stable"]
+  pull_request:
+    branches: ["master", "release/stable"]
+
+timeout: "90" # minutes
+parametrize:
+  matrix:
+    PACKAGE_NAME: ["fabric", "pytorch"]
+machine: "L4_X_2"
+env:
+  MKL_THREADING_LAYER: "GNU"
+  CUDA_LAUNCH_BLOCKING: "1"
+  NCCL_DEBUG: "INFO"
+  TORCHDYNAMO_VERBOSE: "1"
+  FREEZE_REQUIREMENTS: "1"
+  RUN_ONLY_CUDA_TESTS: "1"
+
+run: |
+  # Install Python and UV
+  apt-get update -qq --fix-missing
+  apt-get install -q -y software-properties-common curl
+  # Add deadsnakes PPA for newer Python versions if needed
+  add-apt-repository ppa:deadsnakes/ppa -y
+  apt-get update -qq --fix-missing
+  apt-get install -y python${python_version} python${python_version}-venv python${python_version}-dev
+  ln -sf /usr/bin/python${python_version} /usr/bin/python
+  curl -LsSf https://astral.sh/uv/install.sh | sh
+
+  # Verify UV installation
+  command -v uv || (echo "UV not found in PATH" && exit 1)
+  # Create and activate a local uv virtual environment
+  uv venv .venv -p "/usr/bin/python${python_version}" || uv venv .venv -p "python${python_version}" || uv venv .venv
+  . .venv/bin/activate
+  hash -r
+
+  whereis nvidia
+  nvidia-smi
+  python --version
+  uv --version
+  uv pip list
+  set -ex
+
+  # Parse CUDA version from image tag, e.g., "nvidia/cuda:12.6.3-devel-ubuntu22.04"
+  IMAGE_TAG="${image##*:}"  # "12.6.3-devel-ubuntu22.04"
+  CUDA_VERSION="${IMAGE_TAG%%-*}"  # "12.6.3"
+  echo "Using CUDA version: ${CUDA_VERSION}"
+  CUDA_VERSION_M_M="${CUDA_VERSION%.*}"  # "12.6"
+  CUDA_VERSION_MM="${CUDA_VERSION_M_M//./}"  # "126"
+  export UV_TORCH_BACKEND=cu${CUDA_VERSION_MM}
+
+  # Adjust tests
+  pip install -U -q -r .actions/requirements.txt
+  python .actions/assistant.py copy_replace_imports --source_dir="./tests" \
+    --source_import="lightning.fabric,lightning.pytorch" \
+    --target_import="lightning_fabric,pytorch_lightning"
+
+  # Install package
+  uv pip install ".[dev]"
+
+  # Env details
+  python requirements/collect_env_details.py
+  python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu >= 2, f'GPU: {mgpu}'"
+
+  cd tests/
+  # Testing: benchmarks
+  export PL_RUNNING_BENCHMARKS=1
+  python -m pytest parity_${PACKAGE_NAME} -v --durations=0
+  export PL_RUNNING_BENCHMARKS=0
+
+  # Testing: fabric standalone tasks
+  export PL_RUN_STANDALONE_TESTS=1
+  if [ "${PACKAGE_NAME}" == "fabric" ]; then
+    cd parity_fabric/
+    bash run_standalone_tasks.sh
+    cd ..
+  fi
+  export PL_RUN_STANDALONE_TESTS=0
+
+  cd ..
+  echo "Benchmarks completed successfully"