Merge pull request #1011 from morotti/rmorotti-compatibility

compatibility with newer numpy/pandas

Author: morotti

.circleci/config.yml

@@ -51,10 +51,10 @@ defaults: &defaults
          virtualenv ci
          . ci/bin/activate
          python setup.py develop
-         pip install --upgrade pip
+         pip install --upgrade pip==21.0.1 setuptools==58.0.4
          pip install pytest-server-fixtures[mongodb]
-         echo y | pip uninstall tzlocal # because circleci will install >5.0b
-         pip install tzlocal==4.2 # works for python 3.6 and 3.8
+         pip uninstall -y tzlocal # because circleci will install >5.0b
+         pip install $TEST_PACKAGES
          pip freeze
          python --version
     # Save dependency cache
@@ -95,6 +95,7 @@ jobs:
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_6
       VERSION: $VERSION
       IMAGE_NAME: mangroup/arctic
+      TEST_PACKAGES: "tzlocal==4.2 numpy==1.14.6 pandas==0.25.3"
     working_directory: ~/arctic_3_6
     docker:
     - image: cimg/python:3.6-node
@@ -107,6 +108,7 @@ jobs:
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_7
       VERSION: $VERSION
       IMAGE_NAME: mangroup/arctic
+      TEST_PACKAGES: "tzlocal==4.2 numpy==1.18.5 pandas==1.0.5"
     working_directory: ~/arctic_3_7
     docker:
     - image: cimg/python:3.7-node
@@ -119,6 +121,7 @@ jobs:
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_8
       VERSION: $VERSION
       IMAGE_NAME: mangroup/arctic
+      TEST_PACKAGES: "tzlocal==4.2 numpy==1.21.6 pandas==1.3.5"
     working_directory: ~/arctic_3_8
     docker:
     - image: cimg/python:3.8-node

CHANGES.md

@@ -1,5 +1,17 @@
 ## Changelog
 
+### HEAD
+ * Compatibility: update pinning from pandas<1.1, to pandas<2
+ * Compatibility: update pinning from numpy<1.19 to numpy<2
+ * Build: specify which version of python/numpy/pandas to test
+ * Bugfix: #887 #928 resolve import error on pandas.Panel, Panel was removed in pandas 1.0
+ * Bugfix: fix pandas get_timezone() argument cannot be None
+ * Bugfix: fix np.float/np.int/np.book depreciations
+ * Bugfix: fix numpy tostring() is deprecated. Use tobytes() instead.
+ * Bugfix: numpy.testing.utils is deprecated
+ * Bugfix: pandas.util.testing is deprecated
+ * Cleanup: remove async_benchmark
+
 ### 1.81.2 (2023-05-11)
   * Feature: #1000, #1001, #1002, #1003, #1007, #1008 ArcticDB docs, links etc
   * Bugfix:  #1009 force CircleCI to install tzlocal==4.2

arctic/__init__.py

@@ -1,5 +1,7 @@
 """ The Arctic TimeSeries and Tick store."""
 
+import pandas
+
 from .arctic import Arctic, register_library_type
 from .arctic import VERSION_STORE, TICK_STORE, CHUNK_STORE
 from .store._ndarray_store import NdarrayStore
@@ -24,5 +26,8 @@
 
 register_versioned_storage(PandasDataFrameStore)
 register_versioned_storage(PandasSeriesStore)
-register_versioned_storage(PandasPanelStore)
 register_versioned_storage(NdarrayStore)
+
+if pandas.__version__.startswith("0."):
+    # Panel is removed in pandas 1
+    register_versioned_storage(PandasPanelStore)

arctic/_util.py

@@ -3,10 +3,7 @@
 import numpy as np
 import pymongo
 from pandas import DataFrame
-try:
-    from pandas.testing import assert_frame_equal
-except ImportError:
-    from pandas.util.testing import assert_frame_equal
+from pandas.testing import assert_frame_equal
 
 from ._config import FW_POINTERS_CONFIG_KEY, FwPointersCfg
 

arctic/serialization/incremental.py

@@ -19,9 +19,9 @@
 log = logging.getLogger(__name__)
 
 
-def incremental_checksum(item, curr_sha=None, is_bytes=False):
+def incremental_checksum(item, curr_sha=None):
     curr_sha = hashlib.sha1() if curr_sha is None else curr_sha
-    curr_sha.update(item if is_bytes else item.tostring())
+    curr_sha.update(item)
     return curr_sha
 
 
@@ -182,7 +182,7 @@ def checksum(self, from_idx, to_idx):
             for chunk_bytes, dtype in self.generator_bytes(from_idx=from_idx, to_idx=to_idx):
                 # TODO: what about compress_array here in batches?
                 compressed_chunk = compress(chunk_bytes)
-                total_sha = incremental_checksum(compressed_chunk, curr_sha=total_sha, is_bytes=True)
+                total_sha = incremental_checksum(compressed_chunk, curr_sha=total_sha)
             self._checksum = Binary(total_sha.digest())
         return self._checksum
 
@@ -220,7 +220,7 @@ def _generator(self, from_idx, to_idx, get_bytes=False):
                 forced_dtype=self.dtype if self._has_string_object else None)
 
             # Let the gc collect the intermediate serialized chunk as early as possible
-            chunk = chunk.tostring() if chunk is not None and get_bytes else chunk
+            chunk = chunk.tobytes() if chunk is not None and get_bytes else chunk
 
             yield chunk, self.dtype, from_idx, curr_stop
             from_idx = curr_stop

arctic/serialization/numpy_arrays.py

@@ -115,8 +115,8 @@ def docify(self, df):
                 arr, mask = self._convert_types(df[c].values)
                 dtypes[str(c)] = arr.dtype.str
                 if mask is not None:
-                    masks[str(c)] = Binary(compress(mask.tostring()))
-                arrays.append(arr.tostring())
+                    masks[str(c)] = Binary(compress(mask.tobytes()))
+                arrays.append(arr.tobytes())
             except Exception as e:
                 typ = infer_dtype(df[c], skipna=False)
                 msg = "Column '{}' type is {}".format(str(c), typ)

arctic/serialization/numpy_records.py

@@ -63,9 +63,23 @@ def _multi_index_to_records(index, empty_index):
             log.info("Level in MultiIndex has no name, defaulting to %s" % index_names[i])
     index_tz = []
     for i in index.levels:
-        if isinstance(i, DatetimeIndex):
-            tmp = get_timezone(i.tz)
-            index_tz.append(str(tmp) if tmp is not None else None)
+        if isinstance(i, DatetimeIndex) and i.tz is not None:
+            if PD_VER < '1.1.0':
+                tmp = get_timezone(i.tz)
+                index_tz.append(str(tmp))
+            else:
+                # see implementation of get_timezone() in github
+                # https://github.com/pandas-dev/pandas/blob/v0.25.3/pandas/_libs/tslibs/timezones.pyx
+                # https://github.com/pandas-dev/pandas/blob/v1.0.5/pandas/_libs/tslibs/timezones.pyx
+                # https://github.com/pandas-dev/pandas/blob/v1.3.5/pandas/_libs/tslibs/timezones.pyx
+                # get_timezone(mktz("UTC")) returns 'dateutil//usr/share/zoneinfo/UTC' string before pandas 1.3
+                # get_timezone(mktz("UTC")) returns the timezone object in pandas 1.3+
+                # we need a stable string to serialize and to deserialize,
+                # so let's always encode the filename like pandas is doing for standard datetime objects.
+                # NOTE: constrary to what it looks like, the path is not dependent on the machine,
+                #       there's logic in pandas and datetime to strip the directory prefix.
+                tmp = "dateutil/" + i.tz._filename
+                index_tz.append(str(tmp))
         else:
             index_tz.append(None)
 

arctic/store/_ndarray_store.py

@@ -445,7 +445,7 @@ def append(self, arctic_lib, version, symbol, item, previous_version, dtype=None
             self._do_append(collection, version, symbol, item, previous_version, dirty_append)
 
     def _do_append(self, collection, version, symbol, item, previous_version, dirty_append):
-        data = item.tostring()
+        data = item.tobytes()
         # Compatibility with Arctic 1.22.0 that didn't write base_sha into the version document
         version['base_sha'] = previous_version.get('base_sha', Binary(b''))
         version['up_to'] = previous_version['up_to'] + len(item)
@@ -596,7 +596,7 @@ def check_written(collection, symbol, version):
 
     def checksum(self, item):
         sha = hashlib.sha1()
-        sha.update(item.tostring())
+        sha.update(item.tobytes())
         return Binary(sha.digest())
 
     def write(self, arctic_lib, version, symbol, item, previous_version, dtype=None):
@@ -656,7 +656,7 @@ def _do_write(self, collection, version, symbol, item, previous_version, segment
 
         # Compress
         idxs = range(int(np.ceil(float(length) / rows_per_chunk)))
-        chunks = [(item[i * rows_per_chunk: (i + 1) * rows_per_chunk]).tostring() for i in idxs]
+        chunks = [(item[i * rows_per_chunk: (i + 1) * rows_per_chunk]).tobytes() for i in idxs]
         compressed_chunks = compress_array(chunks)
 
         # Write

arctic/store/_pandas_ndarray_store.py

@@ -4,13 +4,6 @@
 import numpy as np
 from bson.binary import Binary
 from pandas import DataFrame, Series
-try:
-    # TODO delete when early Pandas version support removed
-    import warnings
-    warnings.filterwarnings("ignore", category=FutureWarning)
-    from pandas import Panel
-except ImportError:
-    pass
 
 from arctic._util import NP_OBJECT_DTYPE
 from arctic.serialization.numpy_records import SeriesSerializer, DataFrameSerializer
@@ -62,7 +55,7 @@ def _segment_index(self, recarr, existing_index, start, new_segments):
                 if start > 0:
                     existing_index_arr = existing_index_arr[existing_index_arr['index'] < start]
                 index = np.concatenate((existing_index_arr, index))
-            return Binary(compress(index.tostring()))
+            return Binary(compress(index.tobytes()))
         elif existing_index:
             raise ArcticException("Could not find datetime64 index in item but existing data contains one")
         return None
@@ -220,6 +213,8 @@ class PandasPanelStore(PandasDataFrameStore):
 
     @staticmethod
     def can_write_type(data):
+        # late import, Panel is removed in pandas 1
+        from pandas import Panel
         return isinstance(data, Panel)
 
     def can_write(self, version, symbol, data):

arctic/store/_version_store_utils.py

@@ -20,9 +20,9 @@ def _split_arrs(array_2d, slices):
     but avoids fancy indexing
     """
     if len(array_2d) == 0:
-        return np.empty(0, dtype=np.object)
+        return np.empty(0, dtype=object)
 
-    rtn = np.empty(len(slices) + 1, dtype=np.object)
+    rtn = np.empty(len(slices) + 1, dtype=object)
     start = 0
     for i, s in enumerate(slices):
         rtn[i] = array_2d[start:s]