Fix CutSampler initialization for newer PyTorch versions (#1543)

* Fix CutSampler initialization for newer PyTorch versions

* Update unit tests for newer python and pytorch versions

* Unfreeze some test package versions

* Remove torchscriptability checks for feature extractors

Author: pzelasko

.github/workflows/unit_tests.yml

@@ -16,12 +16,6 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: "3.8"  # note: no torchaudio
-            torch-install-cmd: "pip install torch==1.12.1 --extra-index-url https://download.pytorch.org/whl/cpu"
-            extra_deps: ""
-          - python-version: "3.9"
-            torch-install-cmd: "pip install torch==2.4 torchaudio==2.4 --extra-index-url https://download.pytorch.org/whl/cpu"
-            extra_deps: "multi-storage-client==0.16.0"
           - python-version: "3.10"  # note: no torchaudio
             torch-install-cmd: "pip install torch==2.5 --extra-index-url https://download.pytorch.org/whl/cpu"
             extra_deps: "multi-storage-client==0.16.0"
@@ -31,6 +25,12 @@ jobs:
           - python-version: "3.12"  # note: no torchaudio
             torch-install-cmd: "pip install torch==2.7 --index-url https://download.pytorch.org/whl/cpu"
             extra_deps: "multi-storage-client==0.16.0"
+          - python-version: "3.13"  # note: no torchaudio
+            torch-install-cmd: "pip install torch==2.9 --index-url https://download.pytorch.org/whl/cpu"
+            extra_deps: "multi-storage-client==0.16.0"
+          - python-version: "3.14"  # note: no torchaudio
+            torch-install-cmd: "pip install torch==2.9 --index-url https://download.pytorch.org/whl/cpu"
+            extra_deps: "multi-storage-client==0.16.0"
 
       fail-fast: false
 

lhotse/dataset/sampling/base.py

@@ -74,9 +74,7 @@ def __init__(
         :param rank: Index of distributed node. We will try to infer it by default.
         :param seed: Random seed used to consistently shuffle the dataset across different processes.
         """
-        super().__init__(
-            data_source=None
-        )  # the "data_source" arg is not used in Sampler...
+        super().__init__()
         self.drop_last = drop_last
         self.shuffle = shuffle
         self.seed = seed

lhotse/dataset/sampling/bucketing.py

@@ -371,7 +371,7 @@ def _create_buckets_equal_duration_single(
 
     See also: :meth:`.create_buckets_from_duration_percentiles`.
     """
-    total_duration = np.sum(c.duration for c in cuts)
+    total_duration = np.sum([c.duration for c in cuts])
     bucket_duration = total_duration / num_buckets
     # Define the order for adding cuts. We start at the beginning, then go to
     # the end, and work our way to the middle. Once in the middle we distribute

setup.py

@@ -170,12 +170,12 @@ def mark_lhotse_version(version: str) -> None:
 
 docs_require = (project_root / "docs" / "requirements.txt").read_text().splitlines()
 tests_require = [
-    "pytest==7.1.3",
-    "pytest-forked==1.4.0",
-    "pytest-xdist==2.5.0",
-    "pytest-cov==4.0.0",
+    "pytest",
+    "pytest-forked",
+    "pytest-xdist",
+    "pytest-cov",
     "flake8==5.0.4",
-    "coverage==6.5.0",
+    "coverage",
     "hypothesis==6.56.0",
     "black==22.3.0",
     "isort==5.10.1",

test/features/test_kaldi_layers.py

@@ -66,54 +66,6 @@ def test_wav2mfcc(deterministic_rng):
     assert y.dtype == torch.float32
 
 
-def test_wav2win_is_torchscriptable(deterministic_rng):
-    x = torch.randn(1, 16000, dtype=torch.float32)
-    t = torch.jit.script(Wav2Win())
-    y, _ = t(x)
-    assert y.shape == torch.Size([1, 100, 400])
-    assert y.dtype == torch.float32
-
-
-def test_wav2fft_is_torchscriptable(deterministic_rng):
-    x = torch.randn(1, 16000, dtype=torch.float32)
-    t = torch.jit.script(Wav2FFT())
-    y = t(x)
-    assert y.shape == torch.Size([1, 100, 257])
-    assert y.dtype == torch.complex64
-
-
-def test_wav2spec_is_torchscriptable(deterministic_rng):
-    x = torch.randn(1, 16000, dtype=torch.float32)
-    t = torch.jit.script(Wav2Spec())
-    y = t(x)
-    assert y.shape == torch.Size([1, 100, 257])
-    assert y.dtype == torch.float32
-
-
-def test_wav2logspec_is_torchscriptable(deterministic_rng):
-    x = torch.randn(1, 16000, dtype=torch.float32)
-    t = torch.jit.script(Wav2LogSpec())
-    y = t(x)
-    assert y.shape == torch.Size([1, 100, 257])
-    assert y.dtype == torch.float32
-
-
-def test_wav2logfilterbank_is_torchscriptable(deterministic_rng):
-    x = torch.randn(1, 16000, dtype=torch.float32)
-    t = torch.jit.script(Wav2LogFilterBank())
-    y = t(x)
-    assert y.shape == torch.Size([1, 100, 80])
-    assert y.dtype == torch.float32
-
-
-def test_wav2mfcc_is_torchscriptable(deterministic_rng):
-    x = torch.randn(1, 16000, dtype=torch.float32)
-    t = torch.jit.script(Wav2MFCC())
-    y = t(x)
-    assert y.shape == torch.Size([1, 100, 13])
-    assert y.dtype == torch.float32
-
-
 def test_strided_waveform_batch_streaming_snip_edges_false(deterministic_rng):
     x = torch.arange(16000).unsqueeze(0)
     window_length = 400