Merge pull request #422 from mabel-dev/0.6.28-release

support sorting by multiple columns

Author: joocer

Makefile

@@ -1,13 +1,32 @@
-lint:
-	python -m pip install --quiet --upgrade pycln isort black
-	python -m pycln .
-	python -m isort .
-	python -m black .
+SRC_DIR := mabel
+PYTHON := python
+UV := $(PYTHON) -m uv
+PIP := $(UV) pip
 
-update:
-	python -m pip install --upgrade pip
-	python -m pip install --upgrade -r requirements.txt
-	python -m pip install --upgrade -r tests/requirements.txt
+define print_green
+	@echo "\033[0;32m$(1)\033[0m"
+endef
+
+define print_blue
+	@echo "\033[0;34m$(1)\033[0m"
+endef
+
+lint: ## Run all linting tools
+	$(call print_blue,"Installing linting tools...")
+	@$(PIP) install --quiet --upgrade pycln isort ruff
+	$(call print_blue,"Cleaning unused imports...")
+	@$(PYTHON) -m pycln .
+	$(call print_blue,"Sorting imports...")
+	@$(PYTHON) -m isort .
+	$(call print_blue,"Formatting code...")
+	@$(PYTHON) -m ruff format $(SRC_DIR)
+	$(call print_green,"Linting complete!")
+
+update: ## Update all dependencies
+	$(call print_blue,"Updating dependencies...")
+	@$(PYTHON) -m pip install --upgrade pip uv
+	@$(UV) pip install --upgrade -r tests/requirements.txt
+	@$(UV) pip install --upgrade -r requirements.txt
 
 test:
 	clear

examples/Reading Data.ipynb

@@ -107,7 +107,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from mabel.data import Reader, STORAGE_CLASS, DictSet, SqlReader\n",
+    "from mabel.data import STORAGE_CLASS, DictSet, SqlReader\n",
     "from mabel.adapters.disk import DiskReader"
    ]
   },

mabel/adapters/database/null_writer.py

@@ -6,7 +6,9 @@
 class NullWriter(BaseDatabaseWriter):
     def commit(self, dataframe: DataFrame):
         # save the data to an interim table
-        naive_inserter = f"INSERT INTO `{self.dataset}` ( {', '.join(dataframe.column_names)} )"
+        naive_inserter = (
+            f"INSERT INTO `{self.dataset}` ( {', '.join(dataframe.column_names)} )"
+        )
         naive_inserter += ",\n".join(f"\t({row})" for row in dataframe) + ";"
 
         # row acts like a tuple when used

mabel/adapters/disk/disk_reader.py

@@ -1,4 +1,3 @@
-from ...data.readers.internals.base_inner_reader import BUFFER_SIZE
 from ...data.readers.internals.base_inner_reader import BaseInnerReader
 
 

mabel/adapters/google/bigquery_writer.py

@@ -47,7 +47,9 @@ def __init__(self, project: str, credentials=None, **kwargs):
 
         super().__init__(**kwargs)
 
-    def _get_table(self, dataset, table, schema, partition_expiration: Optional[int] = None):
+    def _get_table(
+        self, dataset, table, schema, partition_expiration: Optional[int] = None
+    ):
         bigquery_client = bigquery.Client()
         dataset = bigquery_client.dataset(dataset)
 
@@ -62,8 +64,12 @@ def _get_table(self, dataset, table, schema, partition_expiration: Optional[int]
                 bigquery.SchemaField(
                     name="_system_date", field_type="TIMESTAMP", mode=FieldMode.REQUIRED
                 ),
-                bigquery.SchemaField(name="name", field_type="STRING", mode=FieldMode.REQUIRED),
-                bigquery.SchemaField(name="age", field_type="INTEGER", mode=FieldMode.REQUIRED),
+                bigquery.SchemaField(
+                    name="name", field_type="STRING", mode=FieldMode.REQUIRED
+                ),
+                bigquery.SchemaField(
+                    name="age", field_type="INTEGER", mode=FieldMode.REQUIRED
+                ),
             ]
             table = bigquery.Table(table_ref, schema=schema)
 

mabel/adapters/google/google_cloud_storage_reader.py

@@ -48,7 +48,9 @@ def get_blobs_at_path(self, path):
         gcs_bucket = client.get_bucket(bucket)
         blobs = list(client.list_blobs(bucket_or_name=gcs_bucket, prefix=object_path))
 
-        yield from [bucket + "/" + blob.name for blob in blobs if not blob.name.endswith("/")]
+        yield from [
+            bucket + "/" + blob.name for blob in blobs if not blob.name.endswith("/")
+        ]
 
 
 def get_blob(bucket: str = None, blob_name: str = None):

mabel/adapters/google/google_cloud_storage_writer.py

@@ -53,7 +53,9 @@ def commit(self, byte_data, override_blob_name=None):
 
         try:
             blob = self.gcs_bucket.blob(blob_name)
-            self.retry(blob.upload_from_string)(byte_data, content_type="application/octet-stream")
+            self.retry(blob.upload_from_string)(
+                byte_data, content_type="application/octet-stream"
+            )
             return blob_name
         except Exception as err:  # pragma: no cover
             import traceback

mabel/adapters/minio/minio_reader.py

@@ -37,7 +37,9 @@ def get_blobs_at_path(self, path):
         bucket, object_path, _, _ = paths.get_parts(path)
         for cycle_date in dates.date_range(self.start_date, self.end_date):
             cycle_path = paths.build_path(path=object_path, date=cycle_date)
-            blobs = self.minio.list_objects(bucket_name=bucket, prefix=cycle_path, recursive=True)
+            blobs = self.minio.list_objects(
+                bucket_name=bucket, prefix=cycle_path, recursive=True
+            )
 
             yield from [
                 bucket + "/" + blob.object_name

mabel/adapters/minio/minio_writer.py

@@ -13,7 +13,13 @@
 
 class MinIoWriter(BaseInnerWriter):
     def __init__(
-        self, *, end_point: str, access_key: str, secret_key: str, secure: bool = False, **kwargs
+        self,
+        *,
+        end_point: str,
+        access_key: str,
+        secret_key: str,
+        secure: bool = False,
+        **kwargs,
     ):
         if not minio_installed:  # pragma: no cover
             raise MissingDependencyError(
@@ -32,6 +38,8 @@ def commit(self, byte_data, override_blob_name=None):
         else:
             blob_name = self._build_path()
 
-        self.client.put_object(self.bucket, blob_name, io.BytesIO(byte_data), len(byte_data))
+        self.client.put_object(
+            self.bucket, blob_name, io.BytesIO(byte_data), len(byte_data)
+        )
 
         return blob_name

mabel/data/internals/collected_set.py

@@ -33,7 +33,8 @@ def __init__(self, dictset: DictSet, column: str, dedupe: bool = False):
             collections.setdefault(key, []).append(my_item)
         if dedupe:
             collections = {
-                k: {frozenset(i.items()): i for i in v}.values() for k, v in collections.items()
+                k: {frozenset(i.items()): i for i in v}.values()
+                for k, v in collections.items()
             }
 
         self._collections = collections
@@ -52,7 +53,9 @@ def count(self, collection=None):
             return {x: len(y) for x, y in self._collections.items()}
         else:
             try:
-                return [len(y) for x, y in self._collections.items() if x == collection].pop()
+                return [
+                    len(y) for x, y in self._collections.items() if x == collection
+                ].pop()
             except IndexError:
                 return 0
 
@@ -72,7 +75,9 @@ def aggregate(self, column, method):
         """
         response = {}
         for key, items in self._collections.items():
-            values = [item.get(column) for item in items if item.get(column) is not None]
+            values = [
+                item.get(column) for item in items if item.get(column) is not None
+            ]
             response[key] = method(values)
         return response