chore: Replace black formatter with ruff format (#2568)

* reformat files using ruff format

* replace black with ruff format

* remove pydocstyle

* update static checking

* intentionally mess up imports

* fix import

Author: LeonLuttenberger

.github/workflows/static-checking.yml

@@ -32,12 +32,12 @@ jobs:
           python -m pip install poetry
           poetry config virtualenvs.create false --local
           poetry install --all-extras -vvv
-      - name: Black style
-        run: black --check .
-      - name: ruff check
-        run: |
-          ruff . --ignore "PL" --ignore "D"
-          ruff awswrangler
+      - name: Run ruff format
+        run: ruff format --check .
+      - name: Run ruff check on source code
+        run: ruff awswrangler --output-format=github
+      - name: Run ruff check on tests and tutorials
+        run: ruff . --ignore "PL" --ignore "D" --output-format=github
       - name: mypy check
         run: mypy --install-types --non-interactive awswrangler
       - name: Pylint Lint

awswrangler/s3/_write.py

@@ -122,9 +122,10 @@ def _sanitize(
     df = catalog.sanitize_dataframe_columns_names(df=df)
     partition_cols = [catalog.sanitize_column_name(p) for p in partition_cols]
     if bucketing_info:
-        bucketing_info = [
-            catalog.sanitize_column_name(bucketing_col) for bucketing_col in bucketing_info[0]
-        ], bucketing_info[1]
+        bucketing_info = (
+            [catalog.sanitize_column_name(bucketing_col) for bucketing_col in bucketing_info[0]],
+            bucketing_info[1],
+        )
     dtype = {catalog.sanitize_column_name(k): v for k, v in dtype.items()}
     _utils.check_duplicated_columns(df=df)
     return _SanitizeResult(df, dtype, partition_cols, bucketing_info)
@@ -380,9 +381,7 @@ def write(  # pylint: disable=too-many-arguments,too-many-locals,too-many-branch
 
                 if (catalog_table_input is None) and (table_type == "GOVERNED"):
                     self._create_glue_table(**create_table_args)  # pylint: disable=protected-access
-                    create_table_args[
-                        "catalog_table_input"
-                    ] = catalog._get_table_input(  # pylint: disable=protected-access
+                    create_table_args["catalog_table_input"] = catalog._get_table_input(  # pylint: disable=protected-access
                         database=database,
                         table=table,
                         boto3_session=boto3_session,

awswrangler/s3/_write_dataset.py

@@ -20,7 +20,8 @@ def _get_bucketing_series(df: pd.DataFrame, bucketing_info: typing.BucketingInfo
     bucket_number_series = (
         df[bucketing_info[0]]
         # Prevent "upcasting" mixed types by casting to object
-        .astype("O").apply(
+        .astype("O")
+        .apply(
             lambda row: _get_bucket_number(bucketing_info[1], [row[col_name] for col_name in bucketing_info[0]]),
             axis="columns",
         )
@@ -329,9 +330,7 @@ def _to_dataset(
     _logger.debug("Wrote %s paths", len(paths))
     _logger.debug("Created partitions_values: %s", partitions_values)
     if (table_type == "GOVERNED") and (table is not None) and (database is not None):
-        list_add_objects: List[
-            List[Dict[str, Any]]
-        ] = lakeformation._build_table_objects(  # pylint: disable=protected-access
+        list_add_objects: List[List[Dict[str, Any]]] = lakeformation._build_table_objects(  # pylint: disable=protected-access
             paths, partitions_values, use_threads=use_threads, boto3_session=boto3_session
         )
         try:

fix.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 set -ex
 
-black .
-ruff --fix --select "I001" --select "I002" awswrangler
+ruff format .
+ruff --fix awswrangler

poetry.lock

@@ -97,15 +97,11 @@ wheel = "^0.38.1"
 msgpack = "*"
 
 # Lint
-black = "^23.12.0"
 boto3-stubs = {version = "^1.26.151", extras = ["athena", "cleanrooms", "chime", "cloudwatch", "dynamodb", "ec2", "emr", "emr-serverless", "glue", "kms", "lakeformation", "logs", "neptune", "opensearch", "opensearchserverless", "quicksight", "rds", "rds-data", "redshift", "redshift-data", "s3", "secretsmanager", "ssm", "sts", "timestream-query", "timestream-write"]}
 doc8 = "^1.0"
 mypy = "^1.0"
 pylint = "^2.17"
 ruff = "^0.1.8"
-isort = "^5.9.2"
-flake8 = "^5.0.1"
-pydocstyle = "^6.1.1"
 
 # Test
 moto = "^4.1"
@@ -135,32 +131,11 @@ myst-parser = "^1.0.0"
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
-[tool.black]
-line-length = 120
-target-version = ["py38", "py39", "py310", "py311", "py312"]
-extend_exclude = '''
-/(
-    \.eggs
-  | \.git
-  | \.hg
-  | \.mypy_cache
-  | \.tox
-  | \.venv
-  | \.env
-  | _build
-  | buck-out
-  | build
-  | dist
-  | .vscode
-  | dev
-  | .coverage
-)/
-'''
-
 [tool.ruff]
 select = ["D", "E", "F", "I001", "I002", "PL", "W"]
 ignore = ["E501", "PLR2004", "PLR0911", "PLR0912", "PLR0913", "PLR0915"]
-fixable = ["I001"]
+fixable = ["I001", "I002", "W291"]
+extend-include = ["*.ipynb"]
 exclude = [
     ".eggs",
     ".git",
@@ -177,6 +152,9 @@ exclude = [
 line-length = 120
 target-version = "py38"
 
+[tool.ruff.lint]
+exclude = ["*.ipynb"]
+
 [tool.ruff.pydocstyle]
 convention = "numpy"
 

pyproject.toml

@@ -41,17 +41,13 @@ def moto_subnet_id() -> str:
 
         vpc_id = ec2_client.create_vpc(
             CidrBlock="10.0.0.0/16",
-        )[
-            "Vpc"
-        ]["VpcId"]
+        )["Vpc"]["VpcId"]
 
         subnet_id = ec2_client.create_subnet(
             VpcId=vpc_id,
             CidrBlock="10.0.0.0/24",
             AvailabilityZone="us-west-1a",
-        )[
-            "Subnet"
-        ]["SubnetId"]
+        )["Subnet"]["SubnetId"]
 
         yield subnet_id
 

tests/unit/test_moto.py

@@ -28,8 +28,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import awswrangler as wr\n",
-    "import boto3"
+    "import boto3\n",
+    "\n",
+    "import awswrangler as wr"
    ]
   },
   {