[DOP-24001] Migrate from flake8 to ruff (#168)

* [DOP-24001] ruff check over data_rentgen/ folder

* [DOP-24001] update github workflow

* [DOP-24001] update github workflow

* [DOP-24001] Add some igmores

* [DOP-24001] fixes

* [DOP-24001] mypy fixes

* [DOP-24001] ruff fixes

* [DOP-24001] move ruff from actions to precommit-ci

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* [DOP-24001] fix too long line

* [DOP-24001] specify dir in precommit

* [DOP-24001] ruff over tests

* Update data_rentgen/consumer/extractors/dataset.py warning message

Co-authored-by: Maxim Martynov <msmarty5@mts.ru>

* [DOP-24001] ruff over tests

* [DOP-24001] ruff over tests

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Maxim Martynov <msmarty5@mts.ru>

Author: 3 people

.github/workflows/codeql-analysis.yml

@@ -51,13 +51,9 @@ jobs:
         run: |
           poetry install --no-root --all-extras --with dev --without docs,test
 
-      - name: Run flake8
-        run: poetry run flake8 data_rentgen/
-
       - name: Run mypy
         run: poetry run mypy ./data_rentgen --config-file ./pyproject.toml
 
-
   codeql:
     name: CodeQL
     runs-on: ubuntu-latest
@@ -75,7 +71,7 @@ jobs:
         with:
           python-version: ${{ env.DEFAULT_PYTHON }}
 
-    # Initializes the CodeQL tools for scanning.
+      # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v3
         with:

.pre-commit-config.yaml

@@ -63,23 +63,12 @@ repos:
     hooks:
       - id: add-trailing-comma
 
-  - repo: https://github.com/pycqa/autoflake
-    rev: v2.3.1
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.2
     hooks:
-      - id: autoflake
-        args: [--in-place]
-
-  - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 25.1.0
-    hooks:
-      - id: black
-
-  - repo: https://github.com/asottile/blacken-docs
-    rev: 1.19.1
-    hooks:
-      - id: blacken-docs
-        additional_dependencies:
-          - black==24.8.0
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
 
   - repo: https://github.com/pycqa/bandit
     rev: 1.8.3
@@ -91,12 +80,6 @@ repos:
           - -ll
         require_serial: true
 
-  - repo: https://github.com/pycqa/isort
-    rev: 6.0.0
-    hooks:
-      - id: isort
-        name: isort
-
   - repo: https://github.com/IamTheFij/docker-pre-commit
     rev: v3.0.1
     hooks:
@@ -128,14 +111,6 @@ repos:
 
   - repo: local
     hooks:
-      - id: flake8
-        name: flake8
-        entry: flake8
-        language: python
-        types: [python]
-        files: ^data_rentgen/.*$
-        pass_filenames: true
-
       - id: mypy
         name: mypy
         entry: mypy ./data_rentgen --config-file ./pyproject.toml
@@ -151,7 +126,6 @@ repos:
 
 ci:
   skip:
-    - flake8 # checked with Github Actions
     - mypy # checked with Github Actions
     - docker-compose-check # cannot run on pre-commit.ci
     - chmod # failing in pre-commit.ci

data_rentgen/consumer/extractors/__init__.py

@@ -16,17 +16,17 @@
 from data_rentgen.consumer.extractors.schema import extract_schema
 
 __all__ = [
+    "BatchExtractionResult",
+    "connect_dataset_with_symlinks",
+    "extract_batch",
     "extract_column_lineage",
-    "extract_dataset_and_symlinks",
     "extract_dataset",
-    "connect_dataset_with_symlinks",
+    "extract_dataset_and_symlinks",
+    "extract_input",
     "extract_job",
-    "extract_run",
-    "extract_run_minimal",
     "extract_operation",
-    "extract_input",
     "extract_output",
+    "extract_run",
+    "extract_run_minimal",
     "extract_schema",
-    "extract_batch",
-    "BatchExtractionResult",
 ]

data_rentgen/consumer/extractors/batch.py

@@ -41,7 +41,7 @@
 )
 
 
-class BatchExtractionResult:  # noqa: WPS338, WPS214
+class BatchExtractionResult:
     """Track results of batch extraction.
 
     Calling any ``add_*`` method will add DTO item to the result, including nested DTOs,
@@ -74,7 +74,7 @@ def __init__(self):
 
     def __repr__(self):
         return (
-            "ExtractionResult("  # noqa: WPS237
+            "ExtractionResult("
             f"locations={len(self._locations)}, "
             f"datasets={len(self._datasets)}, "
             f"dataset_symlinks={len(self._dataset_symlinks)}, "
@@ -90,7 +90,7 @@ def __repr__(self):
         )
 
     @staticmethod
-    def _add(context: dict[tuple, T], new_item: T) -> dict[tuple, T]:  # noqa: WPS602
+    def _add(context: dict[tuple, T], new_item: T) -> dict[tuple, T]:
         key = new_item.unique_key
         if key in context:
             old_item = context[key]
@@ -130,12 +130,12 @@ def add_operation(self, operation: OperationDTO):
         self._add(self._operations, operation)
         self.add_run(operation.run)
 
-    def add_input(self, input: InputDTO):
-        self._add(self._inputs, input)
-        self.add_operation(input.operation)
-        self.add_dataset(input.dataset)
-        if input.schema:
-            self.add_schema(input.schema)
+    def add_input(self, input_: InputDTO):
+        self._add(self._inputs, input_)
+        self.add_operation(input_.operation)
+        self.add_dataset(input_.dataset)
+        if input_.schema:
+            self.add_schema(input_.schema)
 
     def add_output(self, output: OutputDTO):
         self._add(self._outputs, output)
@@ -196,12 +196,12 @@ def _get_operation(self, operation_key: tuple) -> OperationDTO:
         return operation
 
     def _get_input(self, input_key: tuple) -> InputDTO:
-        input = self._inputs[input_key]
-        input.operation = self._get_operation(input.operation.unique_key)
-        input.dataset = self._get_dataset(input.dataset.unique_key)
-        if input.schema:
-            input.schema = self._get_schema(input.schema.unique_key)
-        return input
+        input_ = self._inputs[input_key]
+        input_.operation = self._get_operation(input_.operation.unique_key)
+        input_.dataset = self._get_dataset(input_.dataset.unique_key)
+        if input_.schema:
+            input_.schema = self._get_schema(input_.schema.unique_key)
+        return input_
 
     def _get_output(self, output_key: tuple) -> OutputDTO:
         output = self._outputs[output_key]
@@ -252,23 +252,23 @@ def users(self) -> list[UserDTO]:
         return list(map(self._get_user, self._users))
 
 
-def extract_batch(events: list[OpenLineageRunEvent]) -> BatchExtractionResult:  # noqa: WPS231
+def extract_batch(events: list[OpenLineageRunEvent]) -> BatchExtractionResult:
     result = BatchExtractionResult()
 
     for event in events:
         if event.job.facets.jobType and event.job.facets.jobType.jobType == OpenLineageJobType.JOB:
             operation = extract_operation(event)
             result.add_operation(operation)
             for input_dataset in event.inputs:
-                input, symlinks = extract_input(operation, input_dataset)
-                result.add_input(input)
+                input_, symlinks = extract_input(operation, input_dataset)
+                result.add_input(input_)
                 for symlink in symlinks:
                     result.add_dataset_symlink(symlink)
 
             for output_dataset in event.outputs:
                 output, symlinks = extract_output(operation, output_dataset)
                 result.add_output(output)
-                for symlink in symlinks:  # noqa: WPS440
+                for symlink in symlinks:
                     result.add_dataset_symlink(symlink)
 
             for dataset in event.inputs + event.outputs:

data_rentgen/consumer/extractors/column_lineage.py

@@ -67,7 +67,7 @@ def extract_column_lineage(operation: OperationDTO, target_dataset: OpenLineageD
 
             column_lineage_key = (source_dataset_dto.unique_key, target_dataset_dto.unique_key)
             for transformation in input_field.transformations:
-                # OL integration for Spark before v1.23 (or with columnLineage.datasetLineageEnabled=false, which is still default)
+                # OL integration for Spark before v1.23 (or with columnLineage.datasetLineageEnabled=false, which is still default)  # noqa: E501
                 # produced INDIRECT lineage for each combination source_column x target_column,
                 # which is amlost the cartesian join. It is VERY expensive to handle, just ignore.
                 # See https://github.com/OpenLineage/OpenLineage/pull/3097

data_rentgen/consumer/extractors/dataset.py

@@ -32,10 +32,10 @@
 def connect_dataset_with_symlinks(
     dataset: DatasetDTO,
     symlink: DatasetDTO,
-    type: OpenLineageSymlinkType,
+    type_: OpenLineageSymlinkType,
 ) -> list[DatasetSymlinkDTO]:
     result = []
-    is_metastore_symlink = type == OpenLineageSymlinkType.TABLE
+    is_metastore_symlink = type_ == OpenLineageSymlinkType.TABLE
 
     result.append(
         DatasetSymlinkDTO(
@@ -76,13 +76,14 @@ def extract_dataset_and_symlinks(dataset: OpenLineageDataset) -> tuple[DatasetDT
     if table_symlinks:
         # We are swapping the dataset with its TABLE symlink to create a cleaner lineage.
         # For example, by replacing an HDFS file with its corresponding Hive table.
-        # This ensures that all operations interact with a single table instead of multiple files (which may represent different partitions).
+        # This ensures that all operations interact with a single table instead of multiple files (which may represent different partitions).  # noqa: E501
         # Discussion on this issue: https://github.com/OpenLineage/OpenLineage/issues/2718
 
         # TODO: add support for multiple TABLE symlinks
         if len(table_symlinks) > 1:
             logger.warning(
-                "Dataset has more than one TABLE symlink. Only the first one will be used for replacement. Symlink name: %s",
+                "Dataset has more than one TABLE symlink. "
+                "Only the first one will be used for replacement. Symlink name: %s",
                 table_symlinks[0].name,
             )
         table_dataset_dto = extract_dataset(table_symlinks[0])

data_rentgen/consumer/extractors/output.py

@@ -13,7 +13,7 @@ def extract_output(
     dataset: OpenLineageOutputDataset,
 ) -> tuple[OutputDTO, list[DatasetSymlinkDTO]]:
     lifecycle_change = dataset.facets.lifecycleStateChange
-    if lifecycle_change:
+    if lifecycle_change:  # noqa: SIM108
         output_type = OutputTypeDTO(lifecycle_change.lifecycleStateChange)
     else:
         output_type = OutputTypeDTO.APPEND

data_rentgen/consumer/extractors/run.py

@@ -84,7 +84,7 @@ def enrich_run_identifiers(run: RunDTO, event: OpenLineageRunEvent) -> RunDTO:
     return run
 
 
-def enrich_run_logs(run: RunDTO, event: OpenLineageRunEvent) -> RunDTO:  # noqa: WPS231
+def enrich_run_logs(run: RunDTO, event: OpenLineageRunEvent) -> RunDTO:  # noqa: C901, PLR0912
     spark_application_details = event.run.facets.spark_applicationDetails
     if spark_application_details:
         if spark_application_details.proxyUrl:
@@ -142,7 +142,7 @@ def enrich_run_logs(run: RunDTO, event: OpenLineageRunEvent) -> RunDTO:  # noqa:
     return run
 
 
-def get_airflow_2_3_plus_dag_run_url(  # noqa: WPS114
+def get_airflow_2_3_plus_dag_run_url(
     namespace: str,
     airflow_dag_run_facet: OpenLineageAirflowDagRunFacet,
 ) -> str:
@@ -153,7 +153,7 @@ def get_airflow_2_3_plus_dag_run_url(  # noqa: WPS114
     return f"{namespace}/dags/{dag_id}/grid?dag_run_id={dag_run_id}"
 
 
-def get_airflow_2_x_dag_run_url(  # noqa: WPS114
+def get_airflow_2_x_dag_run_url(
     namespace: str,
     airflow_dag_run_facet: OpenLineageAirflowDagRunFacet,
 ) -> str:
@@ -163,7 +163,7 @@ def get_airflow_2_x_dag_run_url(  # noqa: WPS114
     return f"{namespace}/graph?dag_id={dag_id}&execution_date={execution_date}"
 
 
-def get_airflow_2_9_plus_task_log_url(  # noqa: WPS114
+def get_airflow_2_9_plus_task_log_url(
     namespace: str,
     airflow_task_run_facet: OpenLineageAirflowTaskRunFacet,
 ) -> str:
@@ -178,7 +178,7 @@ def get_airflow_2_9_plus_task_log_url(  # noqa: WPS114
     return f"{namespace}/dags/{dag_id}/grid?tab=logs&dag_run_id={dag_run_id}&task_id={task_id}&map_index={map_index}"
 
 
-def get_airflow_2_x_task_log_url(  # noqa: WPS114
+def get_airflow_2_x_task_log_url(
     namespace: str,
     airflow_task_run_facet: OpenLineageAirflowTaskRunFacet,
 ) -> str:

data_rentgen/consumer/openlineage/__init__.py

@@ -6,4 +6,4 @@
 So we have to write our own deserialization logic.
 
 Also FastStream support only ``pydantic`` models whether openlineage-python provides ``attrs`` models.
-"""
+"""  # noqa: E501

data_rentgen/consumer/openlineage/dataset_facets/__init__.py

@@ -44,27 +44,27 @@
 )
 
 __all__ = [
+    "OpenLineageColumnLineageDatasetFacet",
+    "OpenLineageColumnLineageDatasetFacetField",
+    "OpenLineageColumnLineageDatasetFacetFieldRef",
+    "OpenLineageColumnLineageDatasetFacetFieldTransformation",
     "OpenLineageDatasetFacet",
+    "OpenLineageDatasetFacets",
+    "OpenLineageDatasetLifecycleStateChange",
+    "OpenLineageDatasetPreviousIdentifier",
     "OpenLineageDatasourceDatasetFacet",
     "OpenLineageDocumentationDatasetFacet",
-    "OpenLineageLifecycleStateChangeDatasetFacet",
-    "OpenLineageDatasetPreviousIdentifier",
-    "OpenLineageDatasetLifecycleStateChange",
+    "OpenLineageInputDatasetFacets",
     "OpenLineageInputStatisticsInputDatasetFacet",
+    "OpenLineageLifecycleStateChangeDatasetFacet",
+    "OpenLineageOutputDatasetFacets",
     "OpenLineageOutputStatisticsOutputDatasetFacet",
     "OpenLineageSchemaDatasetFacet",
     "OpenLineageSchemaField",
     "OpenLineageStorageDatasetFacet",
-    "OpenLineageSymlinksDatasetFacet",
-    "OpenLineageSymlinkType",
     "OpenLineageSymlinkIdentifier",
-    "OpenLineageDatasetFacets",
-    "OpenLineageInputDatasetFacets",
-    "OpenLineageOutputDatasetFacets",
-    "OpenLineageColumnLineageDatasetFacet",
-    "OpenLineageColumnLineageDatasetFacetField",
-    "OpenLineageColumnLineageDatasetFacetFieldRef",
-    "OpenLineageColumnLineageDatasetFacetFieldTransformation",
+    "OpenLineageSymlinkType",
+    "OpenLineageSymlinksDatasetFacet",
 ]