Fix CPU Linux unittest job - part 1 (#3926)

Author: NicolasHug

.github/scripts/unittest-linux/install.sh

@@ -25,38 +25,57 @@ conda create -n ci -y python="${PYTHON_VERSION}"
 conda activate ci
 
 # 1. Install PyTorch
-if [ -z "${CUDA_VERSION:-}" ] ; then
-    if [ "${os}" == MacOSX ] ; then
-        cudatoolkit=''
-    else
-        cudatoolkit="cpuonly"
-    fi
-    version="cpu"
-else
-    version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")"
-    export CUDATOOLKIT_CHANNEL="nvidia"
-    cudatoolkit="pytorch-cuda=${version}"
-fi
+# if [ -z "${CUDA_VERSION:-}" ] ; then
+#     if [ "${os}" == MacOSX ] ; then
+#         cudatoolkit=''
+#     else
+#         cudatoolkit="cpuonly"
+#     fi
+#     version="cpu"
+# else
+#     version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")"
+#     export CUDATOOLKIT_CHANNEL="nvidia"
+#     cudatoolkit="pytorch-cuda=${version}"
+# fi
 
-printf "Installing PyTorch with %s\n" "${cudatoolkit}"
-(
-    if [ "${os}" == MacOSX ] ; then
-      # TODO: this can be removed as soon as linking issue could be resolved
-      #  see https://github.com/pytorch/pytorch/issues/62424 from details
-      MKL_CONSTRAINT='mkl==2021.2.0'
-      pytorch_build=pytorch
-    else
-      MKL_CONSTRAINT=''
-      pytorch_build="pytorch[build="*${version}*"]"
-    fi
-    set -x
+# printf "Installing PyTorch with %s\n" "${cudatoolkit}"
+# (
+#     if [ "${os}" == MacOSX ] ; then
+#       # TODO: this can be removed as soon as linking issue could be resolved
+#       #  see https://github.com/pytorch/pytorch/issues/62424 from details
+#       MKL_CONSTRAINT='mkl==2021.2.0'
+#       pytorch_build=pytorch
+#     else
+#       MKL_CONSTRAINT=''
+#       pytorch_build="pytorch[build="*${version}*"]"
+#     fi
+#     set -x
+
+#     if [[ -z "$cudatoolkit" ]]; then
+#         conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" $MKL_CONSTRAINT "pytorch-${UPLOAD_CHANNEL}::${pytorch_build}"
+#     else
+#         conda install pytorch ${cudatoolkit} ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" -c nvidia  $MKL_CONSTRAINT
+#     fi
+# )
+
+export GPU_ARCH_TYPE="cpu"  # TODO change this
+
+case $GPU_ARCH_TYPE in
+  cpu)
+    GPU_ARCH_ID="cpu"
+    ;;
+  cuda)
+    VERSION_WITHOUT_DOT=$(echo "${GPU_ARCH_VERSION}" | sed 's/\.//')
+    GPU_ARCH_ID="cu${VERSION_WITHOUT_DOT}"
+    ;;
+  *)
+    echo "Unknown GPU_ARCH_TYPE=${GPU_ARCH_TYPE}"
+    exit 1
+    ;;
+esac
+PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/${GPU_ARCH_ID}"
+pip install --progress-bar=off --pre torch --index-url="${PYTORCH_WHEEL_INDEX}"
 
-    if [[ -z "$cudatoolkit" ]]; then
-        conda install ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" $MKL_CONSTRAINT "pytorch-${UPLOAD_CHANNEL}::${pytorch_build}"
-    else
-        conda install pytorch ${cudatoolkit} ${CONDA_CHANNEL_FLAGS:-} -y -c "pytorch-${UPLOAD_CHANNEL}" -c nvidia  $MKL_CONSTRAINT
-    fi
-)
 
 # 2. Install torchaudio
 conda install --quiet -y ninja cmake
@@ -76,8 +95,8 @@ fi
 # Note: installing librosa via pip fail because it will try to compile numba.
 (
     set -x
-    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} sox libvorbis 'librosa==0.10.0' parameterized 'requests>=2.20' 'ffmpeg>=6,<7'
-    pip install kaldi-io SoundFile coverage pytest pytest-cov 'scipy==1.7.3' expecttest unidecode inflect Pillow sentencepiece pytorch-lightning 'protobuf<4.21.0' demucs tinytag pyroomacoustics flashlight-text git+https://github.com/kpu/kenlm
+    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} sox libvorbis librosa parameterized 'requests>=2.20' 'ffmpeg>=6,<7'
+    pip install kaldi-io SoundFile coverage pytest pytest-cov scipy expecttest unidecode inflect Pillow sentencepiece pytorch-lightning 'protobuf<4.21.0' demucs tinytag pyroomacoustics flashlight-text git+https://github.com/kpu/kenlm
 )
 # Install fairseq
 git clone https://github.com/pytorch/fairseq

.github/workflows/unittest-linux-cpu.yml

@@ -13,8 +13,8 @@ jobs:
   tests:
     strategy:
       matrix:
-        # TODO add up to 3.13
-        python_version: ["3.9", "3.10"]
+        # TODO all from 3.9 to 3.13
+        python_version: ["3.11"]
       fail-fast: false
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
     permissions: