diff --git a/.github/scripts/unittest-linux/install.sh b/.github/scripts/unittest-linux/install.sh
index ad4801f8b6..a7ae9bfcf4 100755
--- a/.github/scripts/unittest-linux/install.sh
+++ b/.github/scripts/unittest-linux/install.sh
@@ -98,8 +98,8 @@ if [[ "$(python --version)" = *3.9* || "$(python --version)" = *3.10* ]]; then
 fi
 (
     set -x
-    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} sox libvorbis parameterized 'requests>=2.20'
-    pip install kaldi-io SoundFile librosa coverage pytest pytest-cov scipy expecttest unidecode inflect Pillow sentencepiece pytorch-lightning 'protobuf<4.21.0' demucs tinytag pyroomacoustics flashlight-text git+https://github.com/kpu/kenlm
+    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} libvorbis parameterized 'requests>=2.20'
+    pip install SoundFile coverage pytest pytest-cov scipy expecttest unidecode inflect Pillow sentencepiece pytorch-lightning 'protobuf<4.21.0' demucs tinytag pyroomacoustics flashlight-text git+https://github.com/kpu/kenlm
 
     # TODO: might be better to fix the single call to `pip install` above
     pip install pillow scipy "numpy>=1.26"
diff --git a/.github/scripts/unittest-linux/run_test.sh b/.github/scripts/unittest-linux/run_test.sh
index c5c55ae89c..980973eab0 100755
--- a/.github/scripts/unittest-linux/run_test.sh
+++ b/.github/scripts/unittest-linux/run_test.sh
@@ -30,5 +30,5 @@ fi
 
 (
     cd test
-    pytest torchaudio_unittest -k "not backend and not /io/ and not prototype and not sox and not ffmpeg and not fairseq and not hdemucs and not (torchscript and rnnt) and not torchscript_consistency"
+    pytest torchaudio_unittest -k "not backend and not (sox and not compatibility) and not /io/ and not prototype and not ffmpeg and not fairseq and not hdemucs and not (torchscript and rnnt) and not torchscript_consistency"
 )
diff --git a/.github/scripts/unittest-windows/environment.yml b/.github/scripts/unittest-windows/environment.yml
index 16225c9e37..c550773a38 100644
--- a/.github/scripts/unittest-windows/environment.yml
+++ b/.github/scripts/unittest-windows/environment.yml
@@ -8,7 +8,6 @@ dependencies:
   - scipy >= 1.4.1
   - pip
   - pip:
-    - kaldi-io
     - PySoundFile
     - future
     - parameterized
diff --git a/.github/scripts/unittest-windows/install.sh b/.github/scripts/unittest-windows/install.sh
index a6d2c8beba..dcb9503e2a 100644
--- a/.github/scripts/unittest-windows/install.sh
+++ b/.github/scripts/unittest-windows/install.sh
@@ -64,9 +64,8 @@ case "$(python --version)" in
         NUMBA_DEV_CHANNEL="-c numba/label/dev"
         ;;
 esac
-# Note: installing librosa via pip fail because it will try to compile numba.
 (
-    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} 'librosa==0.10.0' parameterized 'requests>=2.20'
+    conda install -y -c conda-forge ${NUMBA_DEV_CHANNEL} parameterized 'requests>=2.20'
     # Need to disable shell check since this'll fail out if SENTENCEPIECE_DEPENDENCY is empty
     # shellcheck disable=SC2086
     pip install \
@@ -76,7 +75,6 @@ esac
         coverage \
         expecttest \
         inflect \
-        kaldi-io \
         pytest \
         pytest-cov \
         pytorch-lightning \
diff --git a/.github/workflows/unittest-linux-gpu.yml b/.github/workflows/unittest-linux-gpu.yml
index aef573b428..429fb96ab3 100644
--- a/.github/workflows/unittest-linux-gpu.yml
+++ b/.github/workflows/unittest-linux-gpu.yml
@@ -34,11 +34,6 @@ jobs:
         export PYTHON_VERSION="${{ matrix.python_version }}"
         export CU_VERSION="${{ matrix.cuda_arch_version }}"
         export CUDATOOLKIT="pytorch-cuda=${CU_VERSION}"
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_APPLY_CMVN_SLIDING=true
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_FBANK_FEATS=true
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_KALDI_PITCH_FEATS=true
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_MFCC_FEATS=true
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_CMD_COMPUTE_SPECTROGRAM_FEATS=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_CUDA_SMALL_MEMORY=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true
         export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
@@ -82,9 +77,8 @@ jobs:
         #   --quiet --yes \
         #   -c conda-forge \
         #   -c numba/label/dev \
-        #   sox libvorbis 'librosa==0.10.0' parameterized 'requests>=2.20'
+        #   libvorbis parameterized 'requests>=2.20'
         # pip3 install --progress-bar off \
-        #   kaldi-io \
         #   SoundFile \
         #   coverage \
         #   pytest \
@@ -105,7 +99,7 @@ jobs:
         #   git+https://github.com/pytorch/fairseq.git@e47a4c8
         
         pip3 install parameterized requests
-        pip3 install kaldi-io SoundFile librosa coverage pytest pytest-cov scipy expecttest unidecode inflect Pillow sentencepiece pytorch-lightning 'protobuf<4.21.0' demucs tinytag 
+        pip3 install SoundFile coverage pytest pytest-cov scipy expecttest unidecode inflect Pillow sentencepiece pytorch-lightning 'protobuf<4.21.0' demucs tinytag 
         pip3 install pillow scipy "numpy>=1.26"
 
         echo "::endgroup::"
diff --git a/examples/hubert/utils/kmeans.py b/examples/hubert/utils/kmeans.py
index eacb635836..c661c53dc9 100644
--- a/examples/hubert/utils/kmeans.py
+++ b/examples/hubert/utils/kmeans.py
@@ -8,7 +8,6 @@
 from typing import Tuple
 
 import torch
-from sklearn.cluster import MiniBatchKMeans
 from torch import Tensor
 
 from .common_utils import _get_feat_lens_paths, _get_model_path
@@ -102,6 +101,7 @@ def learn_kmeans(
     """
     if not km_dir.exists():
         km_dir.mkdir()
+    from sklearn.cluster import MiniBatchKMeans
 
     km_model = MiniBatchKMeans(
         n_clusters=n_clusters,
diff --git a/src/torchaudio/_internal/module_utils.py b/src/torchaudio/_internal/module_utils.py
index 6d6f8c9154..45956cb175 100644
--- a/src/torchaudio/_internal/module_utils.py
+++ b/src/torchaudio/_internal/module_utils.py
@@ -97,7 +97,10 @@ def decorator(func):
     {func.__doc__}
     """
 
-        UNSUPPORTED.append(wrapped)
+        # This is a temporary fix to avoid depending on sox during testing.
+        # It will be removed once the sox dependency is removed from the rest of the codebase.
+        if 'sox' not in func.__module__:
+            UNSUPPORTED.append(wrapped)
         return wrapped
 
     return decorator
diff --git a/test/torchaudio_unittest/assets/kaldi_expected_results/test/torchaudio_unittest/functional/kaldi_compatibility_test.py__TestKaldiFloat32__test_sliding_window_cmn.pt b/test/torchaudio_unittest/assets/kaldi_expected_results/test/torchaudio_unittest/functional/kaldi_compatibility_test.py__TestKaldiFloat32__test_sliding_window_cmn.pt
new file mode 100644
index 0000000000..d9939d6fc5
Binary files /dev/null and b/test/torchaudio_unittest/assets/kaldi_expected_results/test/torchaudio_unittest/functional/kaldi_compatibility_test.py__TestKaldiFloat32__test_sliding_window_cmn.pt differ
diff --git a/test/torchaudio_unittest/assets/kaldi_expected_results/test/torchaudio_unittest/functional/kaldi_compatibility_test.py__TestKaldiFloat64__test_sliding_window_cmn.pt b/test/torchaudio_unittest/assets/kaldi_expected_results/test/torchaudio_unittest/functional/kaldi_compatibility_test.py__TestKaldiFloat64__test_sliding_window_cmn.pt
new file mode 100644
index 0000000000..e57c215b66
Binary files /dev/null and b/test/torchaudio_unittest/assets/kaldi_expected_results/test/torchaudio_unittest/functional/kaldi_compatibility_test.py__TestKaldiFloat64__test_sliding_window_cmn.pt differ
diff --git a/test/torchaudio_unittest/common_utils/__init__.py b/test/torchaudio_unittest/common_utils/__init__.py
index ff58db8f6c..509d5208df 100644
--- a/test/torchaudio_unittest/common_utils/__init__.py
+++ b/test/torchaudio_unittest/common_utils/__init__.py
@@ -15,9 +15,6 @@
     skipIfNoModule,
     skipIfNoQengine,
     skipIfNoRIR,
-    skipIfNoSox,
-    skipIfNoSoxDecoder,
-    skipIfNoSoxEncoder,
     skipIfPy310,
     skipIfRocm,
     TempDirMixin,
@@ -63,9 +60,6 @@ def inject_request(self, request):
     "skipIfNoMacOS",
     "skipIfNoModule",
     "skipIfNoRIR",
-    "skipIfNoSox",
-    "skipIfNoSoxDecoder",
-    "skipIfNoSoxEncoder",
     "skipIfRocm",
     "skipIfNoQengine",
     "skipIfNoFFmpeg",
diff --git a/test/torchaudio_unittest/common_utils/case_utils.py b/test/torchaudio_unittest/common_utils/case_utils.py
index ae8ab05cee..b99b96f5b0 100644
--- a/test/torchaudio_unittest/common_utils/case_utils.py
+++ b/test/torchaudio_unittest/common_utils/case_utils.py
@@ -206,28 +206,6 @@ def skipIfNoModule(module, display_name=None):
     reason="CUDA does not have enough memory.",
     key="CUDA_SMALL_MEMORY",
 )
-skipIfNoSox = _skipIf(
-    not _IS_SOX_AVAILABLE,
-    reason="Sox features are not available.",
-    key="NO_SOX",
-)
-
-
-def skipIfNoSoxDecoder(ext):
-    return _skipIf(
-        not _IS_SOX_AVAILABLE or ext not in torchaudio.utils.sox_utils.list_read_formats(),
-        f'sox does not handle "{ext}" for read.',
-        key="NO_SOX_DECODER",
-    )
-
-
-def skipIfNoSoxEncoder(ext):
-    return _skipIf(
-        not _IS_SOX_AVAILABLE or ext not in torchaudio.utils.sox_utils.list_write_formats(),
-        f'sox does not handle "{ext}" for write.',
-        key="NO_SOX_ENCODER",
-    )
-
 
 skipIfNoRIR = _skipIf(
     not torchaudio._extension._IS_RIR_AVAILABLE,
diff --git a/test/torchaudio_unittest/compliance/kaldi/kaldi_compatibility_impl.py b/test/torchaudio_unittest/compliance/kaldi/kaldi_compatibility_impl.py
index 9781cc4e2a..83b7d6314a 100644
--- a/test/torchaudio_unittest/compliance/kaldi/kaldi_compatibility_impl.py
+++ b/test/torchaudio_unittest/compliance/kaldi/kaldi_compatibility_impl.py
@@ -5,7 +5,6 @@
     get_asset_path,
     load_params,
     load_wav,
-    skipIfNoExec,
     TempDirMixin,
     TestBaseMixin,
     RequestMixin
@@ -19,7 +18,6 @@ def assert_equal(self, output, *, expected, rtol=None, atol=None):
         self.assertEqual(output, expected, rtol=rtol, atol=atol)
 
     @parameterized.expand(load_params("kaldi_test_fbank_args.jsonl"))
-    @skipIfNoExec("compute-fbank-feats")
     def test_fbank(self, kwargs):
         """fbank should be numerically compatible with compute-fbank-feats"""
         wave_file = get_asset_path("kaldi_file.wav")
@@ -30,7 +28,6 @@ def test_fbank(self, kwargs):
         self.assert_equal(result, expected=kaldi_result, rtol=1e-4, atol=1e-8)
 
     @parameterized.expand(load_params("kaldi_test_spectrogram_args.jsonl"))
-    @skipIfNoExec("compute-spectrogram-feats")
     def test_spectrogram(self, kwargs):
         """spectrogram should be numerically compatible with compute-spectrogram-feats"""
         wave_file = get_asset_path("kaldi_file.wav")
@@ -41,7 +38,6 @@ def test_spectrogram(self, kwargs):
         self.assert_equal(result, expected=kaldi_result, rtol=1e-4, atol=1e-6)
 
     @parameterized.expand(load_params("kaldi_test_mfcc_args.jsonl"))
-    @skipIfNoExec("compute-mfcc-feats")
     def test_mfcc(self, kwargs):
         """mfcc should be numerically compatible with compute-mfcc-feats"""
         wave_file = get_asset_path("kaldi_file.wav")
diff --git a/test/torchaudio_unittest/deprecation_test.py b/test/torchaudio_unittest/deprecation_test.py
index 04493c8dc3..c44d1907f9 100644
--- a/test/torchaudio_unittest/deprecation_test.py
+++ b/test/torchaudio_unittest/deprecation_test.py
@@ -3,7 +3,7 @@
 import torch
 
 from torchaudio._internal.module_utils import UNSUPPORTED
-from torchaudio.sox_effects import apply_effects_tensor
+from torchaudio.prototype.functional import exp_sigmoid
 
 # Importing prototype modules is needed to trigger the registration of the
 # corresponding APIs in the UNSUPPORTED register.
@@ -25,10 +25,8 @@ def test_deprecations(func):
 # deprecated for years.
 @pytest.mark.parametrize("scripted", (True, False))
 def test_torchscript_fails(scripted):
-    f = apply_effects_tensor
+    f = exp_sigmoid
     if scripted:
         pytest.xfail("Deprecation decorator breaks torchscript")
         f = torch.jit.script(f)
-    _, out_sample_rate = f(torch.rand(2, 1000), sample_rate=16_000, effects=[["rate", "8000"]])
-    assert out_sample_rate == 8000
-
+    f(torch.rand(2, 1000))
diff --git a/test/torchaudio_unittest/functional/functional_cpu_test.py b/test/torchaudio_unittest/functional/functional_cpu_test.py
index 7b81cc92ac..7168d685ea 100644
--- a/test/torchaudio_unittest/functional/functional_cpu_test.py
+++ b/test/torchaudio_unittest/functional/functional_cpu_test.py
@@ -4,7 +4,7 @@
 import torchaudio.functional as F
 from parameterized import parameterized
 import unittest
-from torchaudio_unittest.common_utils import PytorchTestCase, skipIfNoSox, TorchaudioTestCase
+from torchaudio_unittest.common_utils import PytorchTestCase, TorchaudioTestCase
 
 from .functional_impl import Functional, FunctionalCPUOnly
 
@@ -24,7 +24,6 @@ class TestFunctionalFloat64(Functional, PytorchTestCase):
 
 
 @unittest.skip("deprecated")
-@skipIfNoSox
 class TestApplyCodec(TorchaudioTestCase):
     def _smoke_test(self, format, compression, check_num_frames):
         """
diff --git a/test/torchaudio_unittest/functional/kaldi_compatibility_test_impl.py b/test/torchaudio_unittest/functional/kaldi_compatibility_test_impl.py
index c47bf5210f..04caed7398 100644
--- a/test/torchaudio_unittest/functional/kaldi_compatibility_test_impl.py
+++ b/test/torchaudio_unittest/functional/kaldi_compatibility_test_impl.py
@@ -1,6 +1,6 @@
 import torch
 import torchaudio.functional as F
-from torchaudio_unittest.common_utils import skipIfNoExec, TempDirMixin, TestBaseMixin, RequestMixin
+from torchaudio_unittest.common_utils import TempDirMixin, TestBaseMixin, RequestMixin
 from torchaudio_unittest.common_utils.kaldi_utils import convert_args, run_kaldi
 
 
@@ -9,7 +9,6 @@ def assert_equal(self, output, *, expected, rtol=None, atol=None):
         expected = expected.to(dtype=self.dtype, device=self.device)
         self.assertEqual(output, expected, rtol=rtol, atol=atol)
 
-    @skipIfNoExec("apply-cmvn-sliding")
     def test_sliding_window_cmn(self):
         """sliding_window_cmn should be numerically compatible with apply-cmvn-sliding"""
         kwargs = {
@@ -19,7 +18,8 @@ def test_sliding_window_cmn(self):
             "norm_vars": False,
         }
 
-        tensor = torch.randn(40, 10, dtype=self.dtype, device=self.device)
+        torch.manual_seed(0)
+        tensor = torch.randn(40, 10, dtype=self.dtype).to(self.device)
         result = F.sliding_window_cmn(tensor, **kwargs)
         command = ["apply-cmvn-sliding"] + convert_args(**kwargs) + ["ark:-", "ark:-"]
         kaldi_result = run_kaldi(self.request, command, "ark", tensor)
diff --git a/test/torchaudio_unittest/functional/sox_compatibility_test.py b/test/torchaudio_unittest/functional/sox_compatibility_test.py
index 9ec8383272..0a41ec13ac 100644
--- a/test/torchaudio_unittest/functional/sox_compatibility_test.py
+++ b/test/torchaudio_unittest/functional/sox_compatibility_test.py
@@ -5,8 +5,6 @@
     get_whitenoise,
     load_wav,
     save_wav,
-    skipIfNoExec,
-    skipIfNoSox,
     sox_utils,
     TempDirMixin,
     TorchaudioTestCase,
@@ -14,8 +12,6 @@
 )
 
 
-@skipIfNoSox
-@skipIfNoExec("sox")
 class TestFunctionalFiltering(TempDirMixin, TorchaudioTestCase, RequestMixin):
     def run_sox_effect(self, input_file, effect):
         output_file = self.get_temp_path("expected.wav")
diff --git a/test/torchaudio_unittest/transforms/sox_compatibility_test.py b/test/torchaudio_unittest/transforms/sox_compatibility_test.py
index 222bb463b3..a20c44e5ba 100644
--- a/test/torchaudio_unittest/transforms/sox_compatibility_test.py
+++ b/test/torchaudio_unittest/transforms/sox_compatibility_test.py
@@ -8,8 +8,6 @@
     get_whitenoise,
     load_wav,
     save_wav,
-    skipIfNoExec,
-    skipIfNoSox,
     sox_utils,
     TempDirMixin,
     TorchaudioTestCase,
@@ -17,8 +15,6 @@
 )
 
 
-@skipIfNoSox
-@skipIfNoExec("sox")
 class TestFunctionalFiltering(TempDirMixin, TorchaudioTestCase, RequestMixin):
     def run_sox_effect(self, input_file, effect):
         output_file = self.get_temp_path("expected.wav")
diff --git a/test/torchaudio_unittest/utils/sox_utils_test.py b/test/torchaudio_unittest/utils/sox_utils_test.py
index 8b88d966c3..92fba8dc7d 100644
--- a/test/torchaudio_unittest/utils/sox_utils_test.py
+++ b/test/torchaudio_unittest/utils/sox_utils_test.py
@@ -1,8 +1,7 @@
 from torchaudio.utils import sox_utils
-from torchaudio_unittest.common_utils import PytorchTestCase, skipIfNoSox
+from torchaudio_unittest.common_utils import PytorchTestCase
 
 
-@skipIfNoSox
 class TestSoxUtils(PytorchTestCase):
     """Smoke tests for sox_util module"""