Remove conditions for Minhash #679

Author: praateekmahajan

nemo_curator/_compat.py

@@ -39,18 +39,11 @@
 except (ImportError, TypeError):
     CURRENT_CUDF_VERSION = parse_version("24.10.0")
 
-# TODO: remove this once 25.02 becomes the base version of cudf in nemo-curator
-
-# minhash in < 24.12 used to have a minhash(txt) api which was deprecated in favor of
-# minhash(a, b) in 25.02 (in 24.12, minhash_permuted(a,b) was introduced)
-MINHASH_DEPRECATED_API = CURRENT_CUDF_VERSION.base_version < parse_version("24.12").base_version
-MINHASH_PERMUTED_AVAILABLE = (CURRENT_CUDF_VERSION.major == 24) & (CURRENT_CUDF_VERSION.minor == 12)  # noqa: PLR2004
-
 # TODO: remove when dask min version gets bumped
 DASK_SHUFFLE_METHOD_ARG = _dask_version > parse_version("2024.1.0")
 DASK_P2P_ERROR = _dask_version < parse_version("2023.10.0")
 DASK_SHUFFLE_CAST_DTYPE = _dask_version > parse_version("2023.12.0")
-DASK_CUDF_PARQUET_READ_INCONSISTENT_SCHEMA = _dask_version > parse_version("2024.12")
+DASK_CUDF_PARQUET_READ_INCONSISTENT_SCHEMA = _dask_version > parse_version("2025.2.0")
 
 # Query-planning check (and cache)
 _DASK_QUERY_PLANNING_ENABLED = None

nemo_curator/modules/fuzzy_dedup/minhash.py

@@ -27,7 +27,6 @@
 import dask_cudf
 import numpy as np
 
-from nemo_curator._compat import MINHASH_DEPRECATED_API, MINHASH_PERMUTED_AVAILABLE
 from nemo_curator.datasets import DocumentDataset
 from nemo_curator.log import create_logger
 from nemo_curator.utils.distributed_utils import performance_report_if_with_ts_suffix
@@ -71,14 +70,11 @@ def __init__(  # noqa: PLR0913
         self.num_hashes = num_hashes
         self.char_ngram = char_ngrams
 
-        if MINHASH_DEPRECATED_API:
-            self.seeds = self.generate_seeds(n_seeds=self.num_hashes, seed=seed)
-        else:
-            self.seeds = self.generate_hash_permutation_seeds(
-                bit_width=64 if use_64bit_hash else 32,
-                n_permutations=self.num_hashes,
-                seed=seed,
-            )
+        self.seeds = self.generate_hash_permutation_seeds(
+            bit_width=64 if use_64bit_hash else 32,
+            n_permutations=self.num_hashes,
+            seed=seed,
+        )
 
         self.minhash_method = self.minhash64 if use_64bit_hash else self.minhash32
         self.id_field = id_field
@@ -98,13 +94,6 @@ def __init__(  # noqa: PLR0913
         else:
             self._logger = logger
 
-    def generate_seeds(self, n_seeds: int = 260, seed: int = 0) -> np.ndarray:
-        """
-        Generate seeds for all minhash permutations based on the given seed.
-        """
-        gen = np.random.RandomState(seed)
-        return gen.randint(0, 1e6, size=n_seeds)
-
     def generate_hash_permutation_seeds(self, bit_width: int, n_permutations: int = 260, seed: int = 0) -> np.ndarray:
         """
         Generate seeds for all minhash permutations based on the given seed.
@@ -141,24 +130,10 @@ def minhash32(self, ser: cudf.Series, seeds: np.ndarray, char_ngram: int) -> cud
             msg = "Expected data of type cudf.Series"
             raise TypeError(msg)
 
-        if MINHASH_DEPRECATED_API:
-            warnings.warn(
-                "Using an outdated minhash implementation, please update to cuDF version 24.12 "
-                "or later for improved performance. "
-                "Install the latest version of cuDF using `pip install curator[cuda12x_nightly]`",
-                category=FutureWarning,
-                stacklevel=2,
-            )
-            seeds = cudf.Series(seeds, dtype="uint32")
-            return ser.str.minhash(seeds=seeds, width=char_ngram)
-        else:
-            seeds_a = cudf.Series(seeds[:, 0], dtype="uint32")
-            seeds_b = cudf.Series(seeds[:, 1], dtype="uint32")
+        seeds_a = cudf.Series(seeds[:, 0], dtype="uint32")
+        seeds_b = cudf.Series(seeds[:, 1], dtype="uint32")
 
-            if MINHASH_PERMUTED_AVAILABLE:
-                return ser.str.minhash_permuted(a=seeds_a, b=seeds_b, seed=seeds[0][0], width=char_ngram)
-            else:
-                return ser.str.minhash(a=seeds_a, b=seeds_b, seed=seeds[0][0], width=char_ngram)
+        return ser.str.minhash(a=seeds_a, b=seeds_b, seed=seeds[0][0], width=char_ngram)
 
     def minhash64(self, ser: cudf.Series, seeds: np.ndarray, char_ngram: int) -> cudf.Series:
         """
@@ -167,24 +142,9 @@ def minhash64(self, ser: cudf.Series, seeds: np.ndarray, char_ngram: int) -> cud
         if not isinstance(ser, cudf.Series):
             msg = "Expected data of type cudf.Series"
             raise TypeError(msg)
-        if MINHASH_DEPRECATED_API:
-            warnings.warn(
-                "Using an outdated minhash implementation, please update to cuDF version 24.12 "
-                "or later for improved performance. "
-                "Install the latest version of cuDF using `pip install curator[cuda12x_nightly]`",
-                category=FutureWarning,
-                stacklevel=2,
-            )
-            seeds = cudf.Series(seeds, dtype="uint64")
-            return ser.str.minhash64(seeds=seeds, width=char_ngram)
-        else:
-            seeds_a = cudf.Series(seeds[:, 0], dtype="uint64")
-            seeds_b = cudf.Series(seeds[:, 1], dtype="uint64")
-
-            if MINHASH_PERMUTED_AVAILABLE:
-                return ser.str.minhash64_permuted(a=seeds_a, b=seeds_b, seed=seeds[0][0], width=char_ngram)
-            else:
-                return ser.str.minhash64(a=seeds_a, b=seeds_b, seed=seeds[0][0], width=char_ngram)
+        seeds_a = cudf.Series(seeds[:, 0], dtype="uint64")
+        seeds_b = cudf.Series(seeds[:, 1], dtype="uint64")
+        return ser.str.minhash64(a=seeds_a, b=seeds_b, seed=seeds[0][0], width=char_ngram)
 
     def __call__(self, dataset: DocumentDataset) -> str | DocumentDataset:
         """

tests/test_read_data.py

@@ -567,7 +567,6 @@ def test_read_data_different_columns_files_per_partition(
     assert len(df) == NUM_FILES * NUM_RECORDS
 
 
-@pytest.mark.skip(reason="Parquet tests are failing after upgrading to RAPIDS 25.02")
 @pytest.mark.parametrize(
     ("backend", "file_type"),
     [