fixup! COMPAT: For pandas 0.21 CategoricalDtype

Author: TomAugspurger

dask/array/percentile.py

@@ -17,7 +17,7 @@ def _percentile(a, q, interpolation='linear'):
         return None
     if isinstance(q, Iterator):
         q = list(q)
-    if str(a.dtype) == 'category':
+    if a.dtype.name == 'category':
         result = np.percentile(a.codes, q, interpolation=interpolation)
         import pandas as pd
         return pd.Categorical.from_codes(result, a.categories, a.ordered)
@@ -100,7 +100,7 @@ def merge_percentiles(finalq, qs, vals, Ns, interpolation='lower'):
 
     # TODO: Perform this check above in percentile once dtype checking is easy
     #       Here we silently change meaning
-    if str(vals[0].dtype) == 'category':
+    if vals[0].dtype.name == 'category':
         result = merge_percentiles(finalq, qs, [v.codes for v in vals], Ns, interpolation)
         import pandas as pd
         return pd.Categorical.from_codes(result, vals[0].categories, vals[0].ordered)

dask/dataframe/io/tests/test_io.py

@@ -4,8 +4,7 @@
 
 import pytest
 from threading import Lock
-
-import threading
+from multiprocessing.pool import ThreadPool
 
 import dask.array as da
 import dask.dataframe as dd
@@ -15,7 +14,7 @@
 from dask.utils import tmpfile
 from dask.local import get_sync
 
-from dask.dataframe.utils import assert_eq
+from dask.dataframe.utils import assert_eq, is_categorical_dtype
 
 
 ####################
@@ -119,13 +118,14 @@ def test_from_array_with_record_dtype():
 
 def test_from_bcolz_multiple_threads():
     bcolz = pytest.importorskip('bcolz')
+    pool = ThreadPool(processes=5)
 
-    def check():
+    def check(i):
         t = bcolz.ctable([[1, 2, 3], [1., 2., 3.], ['a', 'b', 'a']],
                          names=['x', 'y', 'a'])
         d = dd.from_bcolz(t, chunksize=2)
         assert d.npartitions == 2
-        assert str(d.dtypes['a']) == 'category'
+        assert is_categorical_dtype(d.dtypes['a'])
         assert list(d.x.compute(get=get_sync)) == [1, 2, 3]
         assert list(d.a.compute(get=get_sync)) == ['a', 'b', 'a']
 
@@ -139,14 +139,7 @@ def check():
         assert (sorted(dd.from_bcolz(t, chunksize=2).dask) !=
                 sorted(dd.from_bcolz(t, chunksize=3).dask))
 
-    threads = []
-    for i in range(5):
-        thread = threading.Thread(target=check)
-        thread.start()
-        threads.append(thread)
-
-    for thread in threads:
-        thread.join()
+    pool.map(check, range(5))
 
 
 def test_from_bcolz():
@@ -156,7 +149,7 @@ def test_from_bcolz():
                      names=['x', 'y', 'a'])
     d = dd.from_bcolz(t, chunksize=2)
     assert d.npartitions == 2
-    assert str(d.dtypes['a']) == 'category'
+    assert is_categorical_dtype(d.dtypes['a'])
     assert list(d.x.compute(get=get_sync)) == [1, 2, 3]
     assert list(d.a.compute(get=get_sync)) == ['a', 'b', 'a']
     L = list(d.index.compute(get=get_sync))