Merge branch 'main' into deprecate-torchtext-tutorials

Author: svekars

.ci/docker/Dockerfile

@@ -0,0 +1,29 @@
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install common dependencies (so that this step can be cached separately)
+COPY ./common/install_base.sh install_base.sh
+RUN bash ./install_base.sh && rm install_base.sh
+
+# Setup user
+# TODO: figure out how to remove this part
+COPY ./common/install_user.sh install_user.sh
+RUN bash ./install_user.sh && rm install_user.sh
+
+COPY ./common/install_docs_reqs.sh install_docs_reqs.sh
+RUN bash ./install_docs_reqs.sh && rm install_docs_reqs.sh
+
+# Install conda and other packages
+ENV ANACONDA_PYTHON_VERSION=3.10
+ENV CONDA_CMAKE yes
+ENV DOCS yes
+ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH
+COPY ./requirements.txt /opt/conda/
+COPY ./common/install_conda.sh install_conda.sh
+COPY ./common/common_utils.sh common_utils.sh
+RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements.txt
+
+USER ci-user
+CMD ["bash"]

.ci/docker/build.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -exu
+
+IMAGE_NAME="$1"
+shift
+
+export UBUNTU_VERSION="20.04"
+export CUDA_VERSION="12.4.1"
+
+export BASE_IMAGE="nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}"
+echo "Building ${IMAGE_NAME} Docker image"
+
+docker build \
+  --no-cache \
+  --progress=plain \
+  -f Dockerfile \
+  --build-arg BASE_IMAGE="${BASE_IMAGE}" \
+  "$@" \
+  .

.ci/docker/common/common_utils.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Work around bug where devtoolset replaces sudo and breaks it.
+as_ci_user() {
+  # NB: unsetting the environment variables works around a conda bug
+  # https://github.com/conda/conda/issues/6576
+  # NB: Pass on PATH and LD_LIBRARY_PATH to sudo invocation
+  # NB: This must be run from a directory that the user has access to,
+  # works around https://github.com/conda/conda-package-handling/pull/34
+  sudo -E -H -u ci-user env -u SUDO_UID -u SUDO_GID -u SUDO_COMMAND -u SUDO_USER env "PATH=$PATH" "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" $*
+}
+
+conda_install() {
+  # Ensure that the install command don't upgrade/downgrade Python
+  # This should be called as
+  #   conda_install pkg1 pkg2 ... [-c channel]
+  as_ci_user conda install -q -n py_$ANACONDA_PYTHON_VERSION -y python="$ANACONDA_PYTHON_VERSION" $*
+}
+
+conda_run() {
+  as_ci_user conda run -n py_$ANACONDA_PYTHON_VERSION --no-capture-output $*
+}
+
+pip_install() {
+  as_ci_user conda run -n py_$ANACONDA_PYTHON_VERSION pip3 install --progress-bar off $*
+}

.ci/docker/common/install_base.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+# Based off of https://github.com/pytorch/pytorch/tree/b52e0bf131a4e55cd987176f9c5a8d2ad6783b4f/.ci/docker
+
+set -ex
+
+install_ubuntu() {
+  # Install common dependencies
+  apt-get update
+  # TODO: Some of these may not be necessary
+  apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    cmake=3.16* \
+    curl \
+    git \
+    wget \
+    sudo \
+    vim \
+    jq \
+    vim \
+    unzip \
+    gdb \
+    rsync \
+    libssl-dev \
+    p7zip-full \
+    libglfw3 \
+    libglfw3-dev \
+    sox \
+    libsox-dev \
+    libsox-fmt-all
+
+  # Cleanup package manager
+  apt-get autoclean && apt-get clean
+  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+  ubuntu)
+    install_ubuntu
+    ;;
+  *)
+    echo "Unable to determine OS..."
+    exit 1
+    ;;
+esac

.ci/docker/common/install_conda.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+set -ex
+
+# Optionally install conda
+if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
+  BASE_URL="https://repo.anaconda.com/miniconda"
+
+  MAJOR_PYTHON_VERSION=$(echo "$ANACONDA_PYTHON_VERSION" | cut -d . -f 1)
+  MINOR_PYTHON_VERSION=$(echo "$ANACONDA_PYTHON_VERSION" | cut -d . -f 2)
+
+  CONDA_FILE="Miniconda3-latest-Linux-x86_64.sh"
+
+  mkdir -p /opt/conda
+  chown ci-user:ci-user /opt/conda
+
+  source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh"
+
+  pushd /tmp
+  wget -q "${BASE_URL}/${CONDA_FILE}"
+  # NB: Manually invoke bash per https://github.com/conda/conda/issues/10431
+  as_ci_user bash "${CONDA_FILE}" -b -f -p "/opt/conda"
+  popd
+
+  # NB: Don't do this, rely on the rpath to get it right
+  #echo "/opt/conda/lib" > /etc/ld.so.conf.d/conda-python.conf
+  #ldconfig
+  sed -e 's|PATH="\(.*\)"|PATH="/opt/conda/bin:\1"|g' -i /etc/environment
+  export PATH="/opt/conda/bin:$PATH"
+
+  # Ensure we run conda in a directory that the user has write access to
+  pushd /opt/conda
+
+  # Prevent conda from updating to 4.14.0, which causes docker build failures
+  # See https://hud.pytorch.org/pytorch/pytorch/commit/754d7f05b6841e555cea5a4b2c505dd9e0baec1d
+  # Uncomment the below when resolved to track the latest conda update
+  # as_ci_user conda update -y -n base conda
+
+  # Install correct Python version
+  as_ci_user conda create -n py_$ANACONDA_PYTHON_VERSION -y python="$ANACONDA_PYTHON_VERSION"
+
+  # Use conda cmake in some cases. Conda cmake will be newer than our supported
+  # min version (3.5 for xenial and 3.10 for bionic), so we only do it in those
+  # following builds that we know should use conda. Specifically, Ubuntu bionic
+  # and focal cannot find conda mkl with stock cmake, so we need a cmake from conda
+  conda_install cmake
+
+  # Install pip packages
+  pip_install -r /opt/conda/requirements.txt
+
+  apt-get update
+  apt-get -y install expect-dev
+
+  popd
+fi

.ci/docker/common/install_docs_reqs.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Based off of https://github.com/pytorch/pytorch/tree/b52e0bf131a4e55cd987176f9c5a8d2ad6783b4f/.ci/docker
+set -ex
+
+apt-get update
+apt-get install -y gpg-agent
+
+curl --retry 3 -sL https://deb.nodesource.com/setup_20.x | sudo -E bash -
+sudo apt-get install -y nodejs
+
+curl --retry 3 -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
+echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
+
+apt-get update
+apt-get install -y --no-install-recommends yarn
+yarn global add katex --prefix /usr/local
+
+sudo apt-get -y install doxygen
+
+apt-get autoclean && apt-get clean
+rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

.ci/docker/common/install_user.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Copied from https://github.com/pytorch/executorch/blob/6e431355a554e5f84c3a05dfa2b981ead90c2b48/.ci/docker/common/install_user.sh#L1
+
+set -ex
+
+# Same as ec2-user
+echo "ci-user:x:1000:1000::/var/lib/ci-user:" >> /etc/passwd
+echo "ci-user:x:1000:" >> /etc/group
+# Needed on Focal or newer
+echo "ci-user:*:19110:0:99999:7:::" >> /etc/shadow
+
+# Create $HOME
+mkdir -p /var/lib/ci-user
+chown ci-user:ci-user /var/lib/ci-user
+
+# Allow sudo
+echo 'ci-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ci-user
+
+# Test that sudo works
+sudo -u ci-user sudo -v

.ci/docker/requirements.txt

@@ -0,0 +1,74 @@
+# --extra-index-url https://download.pytorch.org/whl/cu117/index.html # Use this to run/publish tutorials against the latest binaries during the RC stage. Comment out after the release. Each release verify the correct cuda version.
+# Refer to ./jenkins/build.sh for tutorial build instructions
+
+sphinx==5.0.0
+sphinx-gallery==0.11.1
+sphinx_design
+docutils==0.16
+sphinx-copybutton
+sphinx_sitemap==2.6.0
+pypandoc==1.12
+pandocfilters
+markdown
+tqdm==4.66.1
+numpy==1.24.4
+matplotlib
+librosa
+torch==2.4
+torchvision
+torchtext
+torchdata
+networkx
+PyHamcrest
+bs4
+awscliv2==2.1.1
+flask
+spacy==3.4.1
+ray[tune]==2.7.2
+tensorboard
+jinja2==3.1.3
+pytorch-lightning
+torchx
+torchrl==0.5.0
+tensordict==0.5.0
+ax-platform>=0.4.0
+nbformat>=5.9.2
+datasets
+transformers
+torchmultimodal-nightly # needs to be updated to stable as soon as it's avaialable
+onnx
+onnxscript
+onnxruntime
+evaluate
+accelerate>=0.20.1
+
+importlib-metadata==6.8.0
+
+# PyTorch Theme
+-e git+https://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
+
+ipython
+
+sphinxcontrib.katex
+# to run examples
+boto3
+pandas
+requests
+scikit-image
+scipy==1.11.1
+numba==0.57.1
+pillow==10.2.0
+wget
+gym==0.26.2
+gym-super-mario-bros==7.4.0
+pyopengl
+gymnasium[mujoco]==0.27.0
+timm
+iopath
+pygame==2.6.0
+pycocotools
+semilearn==0.3.2
+torchao==0.5.0
+segment_anything==1.0
+torchrec==0.8.0
+fbgemm-gpu==0.8.0