Run pre-commit to lint files (#305)

jparismorgan · web-flow · commit 9e2aefc9abdc · 2024-04-04T18:34:04.000+02:00
diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml
@@ -5,12 +5,12 @@ on:
   push:
     branches:
       - release-*
-      - '*wheel*'  # must quote since "*" is a YAML reserved character; we want a string
+      - "*wheel*" # must quote since "*" is a YAML reserved character; we want a string
     tags:
-      - '*'
+      - "*"
   pull_request:
     branches:
-      - '*wheel*'  # must quote since "*" is a YAML reserved character; we want a string
+      - "*wheel*" # must quote since "*" is a YAML reserved character; we want a string
 
 jobs:
   generate_backwards_compatibility_data:
@@ -19,7 +19,7 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
-      
+
       # Based on https://github.com/TileDB-Inc/conda-forge-nightly-controller/blob/51519a0f8340b32cf737fcb59b76c6a91c42dc47/.github/workflows/activity.yml#L19C10-L19C10
       - name: Setup git
         run: |
@@ -92,19 +92,19 @@ jobs:
     strategy:
       matrix:
         buildplat:
-          - [ ubuntu-22.04, manylinux_x86_64 ]
-          - [ macos-13, macosx_x86_64 ]
-          - [ macos-13, macosx_arm64 ]
-          - [ windows-2022, win_amd64 ]
-        python: [ "cp39", "cp310", "cp311", "cp312", "pp39" ]
+          - [ubuntu-22.04, manylinux_x86_64]
+          - [macos-13, macosx_x86_64]
+          - [macos-13, macosx_arm64]
+          - [windows-2022, win_amd64]
+        python: ["cp39", "cp310", "cp311", "cp312", "pp39"]
         exclude:
-          - buildplat: [ macos-13, macosx_arm64 ]
+          - buildplat: [macos-13, macosx_arm64]
             python: "pp39"
 
     steps:
       - uses: actions/checkout@v3
 
-      - name: 'Brew setup on macOS' # x-ref c8e49ba8f8b9ce
+      - name: "Brew setup on macOS" # x-ref c8e49ba8f8b9ce
         if: ${{ startsWith(matrix.os, 'macos-') == true }}
         run: |
           set -e pipefail
@@ -128,7 +128,7 @@ jobs:
       - uses: actions/upload-artifact@v4
         with:
           name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
-          path: './wheelhouse/*.whl'
+          path: "./wheelhouse/*.whl"
 
   build_sdist:
     name: Build source distribution
@@ -143,7 +143,6 @@ jobs:
         with:
           path: dist/*.tar.gz
 
-
   upload_pypi:
     needs: [build_wheels, build_sdist]
     runs-on: ubuntu-latest
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -24,6 +24,6 @@ jobs:
       - name: Fix Windows dll inclusion with ctest
         if: ${{ runner.os == 'Windows' }}
         run: |
-            cp .\src\build\externals\install\bin\tiledb.dll D:/a/TileDB-Vector-Search/TileDB-Vector-Search/src/build/libtiledbvectorsearch/include/test
+          cp .\src\build\externals\install\bin\tiledb.dll D:/a/TileDB-Vector-Search/TileDB-Vector-Search/src/build/libtiledbvectorsearch/include/test
       - name: Run Tests
         run: cmake --build ./src/build --target check-ci
diff --git a/apis/python/requirements-py.txt b/apis/python/requirements-py.txt
@@ -1,4 +1,4 @@
 numpy==1.24.3
 tiledb-cloud==0.10.24
 tiledb==0.27.0
-scikit-learn==1.3.2
+scikit-learn==1.3.2
diff --git a/apis/python/src/tiledb/vector_search/__init__.py b/apis/python/src/tiledb/vector_search/__init__.py
@@ -20,12 +20,6 @@
 from .storage_formats import STORAGE_VERSION
 from .storage_formats import storage_formats
 
-from  ._tiledbvspy import FeatureVector
-from  ._tiledbvspy import FeatureVectorArray
-from  ._tiledbvspy import IndexFlatL2
-from  ._tiledbvspy import IndexIVFFlat
-from  ._tiledbvspy import Ctx
-
 try:
     from tiledb.vector_search.version import version as __version__
 except ImportError:
diff --git a/apis/python/src/tiledb/vector_search/embeddings/langchain_embedding.py b/apis/python/src/tiledb/vector_search/embeddings/langchain_embedding.py
@@ -6,7 +6,7 @@
 
 
 # class LangChainEmbedding(ObjectEmbedding):
-class LangChainEmbedding():
+class LangChainEmbedding:
     """
     Embedding functions from `langchain.embeddings` package.
     """
diff --git a/apis/python/src/tiledb/vector_search/index.py b/apis/python/src/tiledb/vector_search/index.py
@@ -4,9 +4,9 @@
 import time
 from typing import Any, Mapping, Optional
 
+from tiledb.vector_search import _tiledbvspy as vspy
 from tiledb.vector_search.module import *
 from tiledb.vector_search.storage_formats import storage_formats
-from tiledb.vector_search import _tiledbvspy as vspy
 
 MAX_UINT64 = np.iinfo(np.dtype("uint64")).max
 MAX_INT32 = np.iinfo(np.dtype("int32")).max
@@ -286,7 +286,10 @@ def check_has_updates(self):
 
     def set_has_updates(self, has_updates: bool = True):
         self.has_updates = has_updates
-        if "has_updates" not in self.group.meta or self.group.meta["has_updates"] != has_updates:
+        if (
+            "has_updates" not in self.group.meta
+            or self.group.meta["has_updates"] != has_updates
+        ):
             self.group.close()
             self.group = tiledb.Group(self.uri, "w", ctx=tiledb.Ctx(self.config))
             self.group.meta["has_updates"] = has_updates
diff --git a/apis/python/src/tiledb/vector_search/module.py b/apis/python/src/tiledb/vector_search/module.py
@@ -4,8 +4,9 @@
 import numpy as np
 
 import tiledb
-from tiledb.vector_search._tiledbvspy import *
 from tiledb.vector_search import _tiledbvspy as vspy
+from tiledb.vector_search._tiledbvspy import *
+
 
 def load_as_matrix(
     path: str,
@@ -35,7 +36,7 @@ def load_as_matrix(
 
     a = tiledb.ArraySchema.load(path, ctx=tiledb.Ctx(config))
     dtype = a.attr(0).dtype
-    # Read all rows from column 0 -> `size`. Set no upper_bound. Note that if `size` is None then 
+    # Read all rows from column 0 -> `size`. Set no upper_bound. Note that if `size` is None then
     # we'll read to the column domain length.
     if dtype == np.float32:
         m = tdbColMajorMatrix_f32(ctx, path, 0, None, 0, size, 0, timestamp)
diff --git a/apis/python/src/tiledb/vector_search/vamana_index.py b/apis/python/src/tiledb/vector_search/vamana_index.py
@@ -1,21 +1,17 @@
-import json
-import multiprocessing
 from typing import Any, Mapping
 
 import numpy as np
-from tiledb.cloud.dag import Mode
 
+from tiledb.vector_search import _tiledbvspy as vspy
 from tiledb.vector_search import index
 from tiledb.vector_search.module import *
-from tiledb.vector_search.storage_formats import (STORAGE_VERSION,
-                                                  storage_formats,
-                                                  validate_storage_version)
-from tiledb.vector_search.utils import add_to_group
-from tiledb.vector_search import _tiledbvspy as vspy
+from tiledb.vector_search.storage_formats import STORAGE_VERSION
+from tiledb.vector_search.storage_formats import storage_formats
 
 MAX_UINT64 = np.iinfo(np.dtype("uint64")).max
 INDEX_TYPE = "VAMANA"
 
+
 class VamanaIndex(index.Index):
     """
     Open a Vamana index
@@ -38,12 +34,16 @@ def __init__(
         super().__init__(uri=uri, config=config, timestamp=timestamp)
         self.index_type = INDEX_TYPE
         self.index = vspy.IndexVamana(vspy.Ctx(config), uri)
-        self.db_uri = self.group[storage_formats[self.storage_version]["PARTS_ARRAY_NAME"]].uri
-        self.ids_uri = self.group[storage_formats[self.storage_version]["IDS_ARRAY_NAME"]].uri
-        
+        self.db_uri = self.group[
+            storage_formats[self.storage_version]["PARTS_ARRAY_NAME"]
+        ].uri
+        self.ids_uri = self.group[
+            storage_formats[self.storage_version]["IDS_ARRAY_NAME"]
+        ].uri
+
         schema = tiledb.ArraySchema.load(self.db_uri, ctx=tiledb.Ctx(self.config))
         self.dimensions = self.index.dimension()
-        
+
         self.dtype = np.dtype(self.group.meta.get("dtype", None))
         if self.dtype is None:
             self.dtype = np.dtype(schema.attr("values").dtype)
@@ -86,6 +86,7 @@ def query_internal(
         # TODO(paris): Actually run the query.
         return [], []
 
+
 # TODO(paris): Pass more arguments to C++, i.e. storage_version.
 def create(
     uri: str,
@@ -98,22 +99,19 @@ def create(
     storage_version: str = STORAGE_VERSION,
     **kwargs,
 ) -> VamanaIndex:
-      if not group_exists:
+    if not group_exists:
         ctx = vspy.Ctx(config)
         index = vspy.IndexVamana(
-            feature_type=np.dtype(vector_type).name, 
-            id_type=np.dtype(id_type).name, 
-            adjacency_row_index_type=np.dtype(adjacency_row_index_type).name, 
+            feature_type=np.dtype(vector_type).name,
+            id_type=np.dtype(id_type).name,
+            adjacency_row_index_type=np.dtype(adjacency_row_index_type).name,
             dimension=dimensions,
         )
         # TODO(paris): Run all of this with a single C++ call.
         empty_vector = vspy.FeatureVectorArray(
-            dimensions, 
-            0, 
-            np.dtype(vector_type).name, 
-            np.dtype(id_type).name
-            )
+            dimensions, 0, np.dtype(vector_type).name, np.dtype(id_type).name
+        )
         index.train(empty_vector)
         index.add(empty_vector)
         index.write_index(ctx, uri)
-      return VamanaIndex(uri=uri, config=config, memory_budget=1000000)
+    return VamanaIndex(uri=uri, config=config, memory_budget=1000000)
diff --git a/apis/python/test/test_api.py b/apis/python/test/test_api.py
@@ -22,6 +22,7 @@ def test_load_matrix(tmpdir):
     assert np.array_equal(m, data)
     assert np.array_equal(orig_matrix[0, 0], data[0, 0])
 
+
 def test_load_matrix_specify_size(tmpdir):
     p = str(tmpdir.mkdir("test").join("test.tdb"))
     data = np.random.rand(12).astype(np.float32).reshape(3, 4)
@@ -41,6 +42,7 @@ def test_load_matrix_specify_size(tmpdir):
     m = vs.load_as_array(p, size=0)
     assert m.shape == (3, 0)
 
+
 def test_vector(tmpdir):
     v = vspy._create_vector_u64()
     assert np.array_equal(np.array(v), np.arange(10))
diff --git a/apis/python/test/test_cloud.py b/apis/python/test/test_cloud.py
@@ -1,8 +1,8 @@
 import os
 import unittest
 
-from common import *
 from array_paths import *
+from common import *
 
 import tiledb.vector_search as vs
 from tiledb.cloud import groups
diff --git a/apis/python/test/test_index.py b/apis/python/test/test_index.py
@@ -1,29 +1,33 @@
+import json
+
 import numpy as np
 import pytest
 from array_paths import *
 from common import *
-import json
 
 import tiledb.vector_search.index as ind
 from tiledb.vector_search import Index
 from tiledb.vector_search import flat_index
 from tiledb.vector_search import ivf_flat_index
 from tiledb.vector_search import vamana_index
-from tiledb.vector_search.index import create_metadata
-from tiledb.vector_search.index import DATASET_TYPE
 from tiledb.vector_search.flat_index import FlatIndex
+from tiledb.vector_search.index import DATASET_TYPE
+from tiledb.vector_search.index import create_metadata
 from tiledb.vector_search.ingestion import ingest
 from tiledb.vector_search.ivf_flat_index import IVFFlatIndex
-from tiledb.vector_search.vamana_index import VamanaIndex
 from tiledb.vector_search.utils import load_fvecs
+from tiledb.vector_search.vamana_index import VamanaIndex
 
 
 def query_and_check(index, queries, k, expected, **kwargs):
     for _ in range(3):
         result_d, result_i = index.query(queries, k=k, **kwargs)
         assert expected.issubset(set(result_i[0]))
 
-def check_default_metadata(uri, expected_vector_type, expected_storage_version, expected_index_type):
+
+def check_default_metadata(
+    uri, expected_vector_type, expected_storage_version, expected_index_type
+):
     group = tiledb.Group(uri, "r", ctx=tiledb.Ctx(None))
     assert "dataset_type" in group.meta
     assert group.meta["dataset_type"] == DATASET_TYPE
@@ -50,9 +54,10 @@ def check_default_metadata(uri, expected_vector_type, expected_storage_version,
     assert type(group.meta["ingestion_timestamps"]) == str
 
     assert "has_updates" in group.meta
-    assert group.meta["has_updates"] == False
+    assert group.meta["has_updates"] == 0
     assert type(group.meta["has_updates"]) == np.int64
 
+
 def test_flat_index(tmp_path):
     uri = os.path.join(tmp_path, "array")
     vector_type = np.dtype(np.uint8)
@@ -161,30 +166,27 @@ def test_ivf_flat_index(tmp_path):
         index, np.array([[2, 2, 2]], dtype=np.float32), 3, {0, 2, 4}, nprobe=partitions
     )
 
+
 def test_vamana_index(tmp_path):
     uri = os.path.join(tmp_path, "array")
     dimensions = 3
     vector_type = np.dtype(np.uint8)
 
     # Create the index.
-    index = vamana_index.create(uri=uri, dimensions=dimensions, vector_type=vector_type, id_type=np.dtype(np.uint32))
-    assert index.get_dimensions() == dimensions
-    query_and_check(
-        index,
-        np.array([[2, 2, 2]], dtype=np.float32),
-        3,
-        {ind.MAX_UINT64}
+    index = vamana_index.create(
+        uri=uri,
+        dimensions=dimensions,
+        vector_type=vector_type,
+        id_type=np.dtype(np.uint32),
     )
+    assert index.get_dimensions() == dimensions
+    query_and_check(index, np.array([[2, 2, 2]], dtype=np.float32), 3, {ind.MAX_UINT64})
 
     # Open the index.
     index = VamanaIndex(uri=uri)
     assert index.get_dimensions() == dimensions
-    query_and_check(
-        index,
-        np.array([[2, 2, 2]], dtype=np.float32),
-        3,
-        {ind.MAX_UINT64}
-    )
+    query_and_check(index, np.array([[2, 2, 2]], dtype=np.float32), 3, {ind.MAX_UINT64})
+
 
 def test_delete_invalid_index(tmp_path):
     # We don't throw with an invalid uri.
@@ -313,6 +315,7 @@ def test_index_with_incorrect_num_of_query_columns_in_single_vector_query(tmp_pa
         with pytest.raises(TypeError):
             index.query(np.array([1, 1, 1], dtype=np.float32), k=3)
 
+
 def test_create_metadata(tmp_path):
     uri = os.path.join(tmp_path, "array")
 
@@ -322,7 +325,9 @@ def test_create_metadata(tmp_path):
     index_type: str = "IVF_FLAT"
     storage_version: str = STORAGE_VERSION
     group_exists: bool = False
-    create_metadata(uri, dimensions, vector_type, index_type, storage_version, group_exists)
-    
+    create_metadata(
+        uri, dimensions, vector_type, index_type, storage_version, group_exists
+    )
+
     # Check it contains the default metadata.
-    check_default_metadata(uri, vector_type, storage_version, index_type)
+    check_default_metadata(uri, vector_type, storage_version, index_type)
diff --git a/apis/python/test/test_object_index.py b/apis/python/test/test_object_index.py
@@ -1,5 +1,5 @@
 from collections import OrderedDict
-from typing import Dict, List, OrderedDict, Tuple
+from typing import Dict, List, Tuple
 
 import numpy as np
 
diff --git a/apis/python/test/test_type_erased_module.py b/apis/python/test/test_type_erased_module.py
diff --git a/src/README.md b/src/README.md
diff --git a/src/include/detail/scoring/README.md b/src/include/detail/scoring/README.md
diff --git a/src/include/index/README.md b/src/include/index/README.md
diff --git a/src/include/test/notebooks/README.md b/src/include/test/notebooks/README.md
