[?] Correct Subarray With Negative Domains on Dense Arrays

* We currently support Python negative indexing ie. where -1 returns
  the last element in the sequence. However, this conflicts with reading
  and writing with negative subarray values -- indexing with negative
  values errors out.
* This PR deprecates the usage of Pythoh negative indexing so that dense
  arrays with negative domains can be indexed by TileDB dimensions
  correctly.

Author: nguyenv

tiledb/libtiledb.pyx

@@ -589,6 +589,8 @@ def index_domain_subarray(array: Array, dom, idx: tuple):
     Return a numpy array representation of the tiledb subarray buffer
     for a given domain and tuple of index slices
     """
+    offsetted = False
+
     ndim = dom.ndim
     if len(idx) != ndim:
         raise IndexError("number of indices does not match domain rank: "
@@ -646,14 +648,11 @@ def index_domain_subarray(array: Array, dom, idx: tuple):
                     raise IndexError("cannot index integral domain dimension with floating point slice")
                 elif not isinstance(start, _inttypes):
                     raise IndexError("cannot index integral domain dimension with non-integral slice (dtype: {})".format(type(start)))
-            # apply negative indexing (wrap-around semantics)
-            if not is_datetime and start < 0:
-                start += int(dim_ub) + 1
             if start < dim_lb:
                 # numpy allows start value < the array dimension shape,
                 # clamp to lower bound of dimension domain
-                #start = dim_lb
-                raise IndexError("index out of bounds <todo>")
+                start = dim_lb
+                # raise IndexError("index out of bounds <todo>")
         else:
             start = dim_lb
         if stop is not None:
@@ -665,8 +664,6 @@ def index_domain_subarray(array: Array, dom, idx: tuple):
                     raise IndexError("cannot index integral domain dimension with floating point slice")
                 elif not isinstance(start, _inttypes):
                     raise IndexError("cannot index integral domain dimension with non-integral slice (dtype: {})".format(type(start)))
-            if not is_datetime and stop < 0:
-                stop += dim_ub
             if stop > dim_ub:
                 # numpy allows stop value > than the array dimension shape,
                 # clamp to upper bound of dimension domain

tiledb/tests/test_libtiledb.py

@@ -464,19 +464,19 @@ def test_array_1d(self):
             assert_array_equal(A, T[slice(None)])
             assert_array_equal(A[:10], T[:10])
             assert_array_equal(A[10:20], T[10:20])
-            assert_array_equal(A[-10:], T[-10:])
+            assert_array_equal(A[-10:], T[1050-10:])
 
             # ellipsis
             assert_array_equal(A[:10, ...], T[:10, ...])
             assert_array_equal(A[10:50, ...], T[10:50, ...])
-            assert_array_equal(A[-50:, ...], T[-50:, ...])
+            assert_array_equal(A[-50:, ...], T[1050-50:, ...])
             assert_array_equal(A[..., :10], T[..., :10])
             assert_array_equal(A[..., 10:20], T[..., 10:20])
-            assert_array_equal(A[..., -50:], T[..., -50:])
+            assert_array_equal(A[..., -50:], T[..., 1050-50:])
 
             # across tiles
             assert_array_equal(A[:150], T[:150])
-            assert_array_equal(A[-250:], T[-250:])
+            assert_array_equal(A[-250:], T[1050-250:])
 
             # point index
             self.assertEqual(A[0], T[0])
@@ -503,7 +503,7 @@ def test_array_1d(self):
             # basic step
             assert_array_equal(A[:50:2], T[:50:2])
             assert_array_equal(A[:2:50], T[:2:50])
-            assert_array_equal(A[10:-1:50], T[10:-1:50])
+            assert_array_equal(A[10:-1:50], T[10:1050:50])
 
             # indexing errors
             with self.assertRaises(IndexError):
@@ -637,46 +637,46 @@ def test_array_2d(self):
             assert_array_equal(A[:10], T[:10])
             assert_array_equal(A[:10], T[:10])
             assert_array_equal(A[10:20], T[10:20])
-            assert_array_equal(A[-10:], T[-10:])
+            assert_array_equal(A[-10:], T[1000-10:])
             assert_array_equal(A[:10, :], T[:10, :])
             assert_array_equal(A[10:20, :], T[10:20, :])
-            assert_array_equal(A[-10:, :], T[-10:, :])
+            assert_array_equal(A[-10:, :], T[1000-10:, :])
             assert_array_equal(A[:10, ...], T[:10, ...])
             assert_array_equal(A[10:20, ...], T[10:20, ...])
-            assert_array_equal(A[-10:, ...], T[-10:, ...])
+            assert_array_equal(A[-10:, ...], T[1000-10:, ...])
             assert_array_equal(A[:10, :, ...], T[:10, :, ...])
             assert_array_equal(A[10:20, :, ...], T[10:20, :, ...])
-            assert_array_equal(A[-10:, :, ...], T[-10:, :, ...])
+            assert_array_equal(A[-10:, :, ...], T[1000-10:, :, ...])
 
             # slice second dimension
             assert_array_equal(A[:, :2], T[:, :2])
             assert_array_equal(A[:, 2:4], T[:, 2:4])
-            assert_array_equal(A[:, -2:], T[:, -2:])
+            assert_array_equal(A[:, -2:], T[:, 10-2:])
             assert_array_equal(A[..., :2], T[..., :2])
             assert_array_equal(A[..., 2:4], T[..., 2:4])
-            assert_array_equal(A[..., -2:], T[..., -2:])
+            assert_array_equal(A[..., -2:], T[..., 10-2:])
             assert_array_equal(A[:, ..., :2], T[:, ..., :2])
             assert_array_equal(A[:, ..., 2:4], T[:, ..., 2:4])
-            assert_array_equal(A[:, ..., -2:], T[:, ..., -2:])
+            assert_array_equal(A[:, ..., -2:], T[:, ..., 10-2:])
 
             # slice both dimensions
             assert_array_equal(A[:10, :2], T[:10, :2])
             assert_array_equal(A[10:20, 2:4], T[10:20, 2:4])
-            assert_array_equal(A[-10:, -2:], T[-10:, -2:])
+            assert_array_equal(A[-10:, -2:], T[1000-10:, 10-2:])
 
             # slice across tile boundries
             assert_array_equal(A[:110], T[:110])
             assert_array_equal(A[190:310], T[190:310])
-            assert_array_equal(A[-110:], T[-110:])
+            assert_array_equal(A[-110:], T[1000-110:])
             assert_array_equal(A[:110, :], T[:110, :])
             assert_array_equal(A[190:310, :], T[190:310, :])
-            assert_array_equal(A[-110:, :], T[-110:, :])
+            assert_array_equal(A[-110:, :], T[1000-110:, :])
             assert_array_equal(A[:, :3], T[:, :3])
             assert_array_equal(A[:, 3:7], T[:, 3:7])
-            assert_array_equal(A[:, -3:], T[:, -3:])
+            assert_array_equal(A[:, -3:], T[:, 10-3:])
             assert_array_equal(A[:110, :3], T[:110, :3])
             assert_array_equal(A[190:310, 3:7], T[190:310, 3:7])
-            assert_array_equal(A[-110:, -3:], T[-110:, -3:])
+            assert_array_equal(A[-110:, -3:], T[1000-110:, 10-3:])
 
             # single row/col/item
             assert_array_equal(A[0], T[0])
@@ -1247,7 +1247,7 @@ def test_varlen_sparse_all_empty_strings(self):
 
         with tiledb.open(uri, mode="r") as T:
             # check interior range
-            assert_array_equal(A[1:-1], T[2:-1]["a1"])
+            assert_array_equal(A[1:-1], T[1 : dim_len - 1]["a1"])
             assert_array_equal(A[1:-1], T.multi_index[2 : dim_len - 1]["a1"])
 
     def test_varlen_write_unicode(self):
@@ -1521,6 +1521,51 @@ def test_array_varlen_2d_s_fixed(self):
         with tiledb.DenseArray(uri) as T:
             assert_array_equal(A, T)
 
+    @pytest.mark.parametrize("lo,hi", [(-20, -10), (-10, 0), (0, 10), (10, 20)])
+    def test_dense_array_with_negative_positive_domain(self, lo, hi):
+        path = self.path("test_dense_array_with_negative_domain")
+        attr = tiledb.Attr(dtype=np.uint8)
+        dom = tiledb.Domain(tiledb.Dim("X", domain=(-20, 20), dtype=np.int64))
+        schema = tiledb.ArraySchema(domain=dom, sparse=False, attrs=[attr])
+        tiledb.Array.create(path, schema)
+        data = np.random.randint(10, size=(hi - lo))
+
+        with tiledb.open(path, "w") as A:
+            A[lo:hi] = data
+
+        with tiledb.open(path, "r") as A:
+            assert_array_equal(A[lo:hi], data[:])
+
+    @pytest.mark.parametrize("lo,hi", [(-20, -10), (-20, -15), (-15, -10), (-17, -13)])
+    def test_dense_array_with_negative_negative_domain(self, lo, hi):
+        path = self.path("test_dense_array_with_negative_negative_domain")
+        attr = tiledb.Attr(dtype=np.uint8)
+        dom = tiledb.Domain(tiledb.Dim("X", domain=(-20, -10), dtype=np.int64))
+        schema = tiledb.ArraySchema(domain=dom, sparse=False, attrs=[attr])
+        tiledb.Array.create(path, schema)
+        data = np.random.randint(10, size=(hi - lo))
+
+        with tiledb.open(path, "w") as A:
+            A[lo:hi] = data
+
+        with tiledb.open(path, "r") as A:
+            assert_array_equal(A[lo:hi], data[:])
+
+    @pytest.mark.parametrize("lo,hi", [(10, 20), (10, 15), (12, 15), (17, 20)])
+    def test_dense_array_with_offset_domain(self, lo, hi):
+        path = self.path("test_dense_array_with_offset_domain")
+        attr = tiledb.Attr(dtype=np.uint8)
+        dom = tiledb.Domain(tiledb.Dim("X", domain=(10, 20), dtype=np.int64))
+        schema = tiledb.ArraySchema(domain=dom, sparse=False, attrs=[attr])
+        tiledb.Array.create(path, schema)
+        data = np.random.randint(10, size=(hi - lo))
+
+        with tiledb.open(path, "w") as A:
+            A[lo:hi] = data
+
+        with tiledb.open(path, "r") as A:
+            assert_array_equal(A[lo:hi], data[:])
+
 
 class TestSparseArray(DiskTestCase):
     @pytest.mark.xfail
@@ -2345,7 +2390,7 @@ def _test_index(self, A, T, idx):
         slice(0, 1050),
         slice(50, 150),
         slice(0, 2000),
-        slice(-150, -50),
+        slice(1050-150, 1050-50),
         # TODO: indexing failures
         # slice(-2000, 2000),
         # slice(0, 0),  # empty result