diff --git a/.github/scripts/unittest-linux/install.sh b/.github/scripts/unittest-linux/install.sh
index a7ae9bfcf4..3e041908c4 100755
--- a/.github/scripts/unittest-linux/install.sh
+++ b/.github/scripts/unittest-linux/install.sh
@@ -98,14 +98,6 @@ if [[ "$(python --version)" = *3.9* || "$(python --version)" = *3.10* ]]; then
 fi
 (
     set -x
-    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} libvorbis parameterized 'requests>=2.20'
-    pip install SoundFile coverage pytest pytest-cov scipy expecttest unidecode inflect Pillow sentencepiece pytorch-lightning 'protobuf<4.21.0' demucs tinytag pyroomacoustics flashlight-text git+https://github.com/kpu/kenlm
-
-    # TODO: might be better to fix the single call to `pip install` above
-    pip install pillow scipy "numpy>=1.26"
+    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} parameterized 'requests>=2.20'
+    pip install coverage pytest pytest-cov scipy expecttest scipy "numpy>=1.26"
 )
-# Install fairseq
-git clone https://github.com/pytorch/fairseq
-cd fairseq
-git checkout e47a4c8
-pip install .
diff --git a/.github/scripts/unittest-linux/run_test.sh b/.github/scripts/unittest-linux/run_test.sh
index 980973eab0..ed7a232eff 100755
--- a/.github/scripts/unittest-linux/run_test.sh
+++ b/.github/scripts/unittest-linux/run_test.sh
@@ -29,6 +29,10 @@ fi
 )
 
 (
+    export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CTC_DECODER=true
+    export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_unidecode=true
+    export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_inflect=true
+    export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_pytorch_lightning=true
     cd test
     pytest torchaudio_unittest -k "not backend and not (sox and not compatibility) and not /io/ and not prototype and not ffmpeg and not fairseq and not hdemucs and not (torchscript and rnnt) and not torchscript_consistency"
 )