Replace darglint with pydoclint (frequenz-floss#124)

The `darglint` project is not maintained anymore, and even when there is
a fork that is trying to keep it alive, it is still extremely slow.

The new project `pydoclint` was created recently but advancing rapidly,
and already in better shape than `darglint` and way faster.

To be able to add ignore comments in the code, `pydoclint` needs to be
run via `flake8`. Because we need to run `flake8` already, and
`pydocstyle` also can run via `flake8`, we are merging both in one
`flake8` call, so we only need to read the files once.

Performance running on the for the SDK `src` directory only:

<table>
<tr>
 <td>darling
 <td>20s (1x)
<tr>
 <td>pydoclint
 <td>0.2s (100x)
<tr>
 <td>flake8 (basic checks + pydocstyle + pydoclint)
 <td>0.6s (33x)
</table>

As a side effect, we are also enabling other base `flake8` checks. There
is probably some overlap with `pylint`, but `flake8` is very fast
anyway, so it shouldn't be noticed (for the SDK, `flake8` basic checks +
`pydocstyle` + `pydoclint` runs in 0.6s and `pylint` alone runs in 15s).
At some point we might want to disable the duplicated checks in `pylint`
to see if it speeds up `pylint`.

This commit also renames the `dev-docstrings` optional dependency to
`dev-flake8` because now is not checking docstrings exclusively.

Author: llucax

.darglint

@@ -1,5 +1,4 @@
 exclude .cookiecutter-replay.json
-exclude .darglint
 exclude .editorconfig
 exclude .gitignore
 exclude CODEOWNERS

MANIFEST.in

@@ -6,25 +6,36 @@
 
 ## Upgrading
 
-<!-- Here goes notes on how to upgrade from previous versions, including deprecations and what they should be replaced with -->
+- `flake8` basic checks are enabled now. Most are already covered by `pylint`, but there are a few differences, so you might need to fix your code if `flake8` find some issues.
+
+- `darglint` was replaced by `pydoclint`, `pydoclint` can find a few more issues than `darglint`, so your code might need adjusting.
+
+- `darglint` is not used anymore, but if it is installed, it will make `flake8` run extremely slowly anyways, so it is extremely recommended to uninstall it (`pip uninstall darglint`) and rebuild you `nox` *venvs* if you use `-R`.
+
+- If you are upgrading without regenerating the cookiecutter templates, you'll need to adjust the dependencies accordingly.
 
 ### Cookiecutter template
 
-<!-- Here upgrade steps for cookiecutter specifically -->
+- CI: The `nox` job now uses a matrix to run the different `nox` sessions in parallel. If you use branch projection with the `nox` job you need to update the rules to include each matrix job.
+
+- See the general upgrading section.
 
 ## New Features
 
-<!-- Here goes the main new features and examples or instructions on how to use them -->
+- `flake8` is now used to check the files.
 
-### Cookiecutter template
+- `darlint` was replaced by `pydoclint`, which is way faster and detect more issues.
 
+### Cookiecutter template
 
 - Now dependabot updates will be done weekly and grouped by *required* and *optional* for minor and patch updates (major updates are still done individually for each dependency).
 
 - ci: Add debug information when installing pip packages.
 
   The output of `pip freeze` is printed to be able to more easily debug different behaviours between GitHub workflow runs and local runs.
 
+- See the general new features section.
+
 ## Bug Fixes
 
 <!-- Here goes notable bug fixes that are worth a special mention or explanation -->

RELEASE_NOTES.md

@@ -230,7 +230,8 @@ def introduction(
 
 Welcome to repo-config Cookiecutter template!
 
-This template will help you to create a new repository for your project. You will be asked to provide some information about your project.
+This template will help you to create a new repository for your project. \
+You will be asked to provide some information about your project.
 
 Here is an explanation of what each variable is for and will be used for:
 

cookiecutter/local_extensions.py

@@ -1,5 +1,4 @@
 exclude .cookiecutter-replay.json
-exclude .darglint
 exclude .editorconfig
 exclude .gitignore
 {%- if cookiecutter.type == "api" %}

cookiecutter/{{cookiecutter.github_repo_name}}/MANIFEST.in

@@ -60,10 +60,12 @@ email = "{{cookiecutter.author_email}}"
 
 # TODO(cookiecutter): Remove and add more optional dependencies if appropriate
 [project.optional-dependencies]
-dev-docstrings = [
+dev-flake8 = [
+  "flake8 == 6.1.0",
+  "flake8-docstrings == 1.7.0",
+  "flake8-pyproject == 1.2.3",  # For reading the flake8 config from pyproject.toml
+  "pydoclint == 0.3.0",
   "pydocstyle == 6.3.0",
-  "darglint == 1.8.1",
-  "tomli == 2.0.1",      # Needed by pydocstyle to read pyproject.toml
 ]
 dev-formatting = ["black == 23.7.0", "isort == 5.12.0"]
 dev-mkdocs = [
@@ -99,7 +101,7 @@ dev-pytest = [
 {%- endif %}
 ]
 dev = [
-  "{{cookiecutter.pypi_package_name}}[dev-mkdocs,dev-docstrings,dev-formatting,dev-mkdocs,dev-mypy,dev-noxfile,dev-pylint,dev-pytest]",
+  "{{cookiecutter.pypi_package_name}}[dev-mkdocs,dev-flake8,dev-formatting,dev-mkdocs,dev-mypy,dev-noxfile,dev-pylint,dev-pytest]",
 ]
 
 [project.urls]
@@ -121,6 +123,23 @@ line_length = 88
 {#- We don't include "py" here for API because we don't want to check generated files #}
 src_paths = ["benchmarks", "examples", "src", "tests"]
 
+[tool.flake8]
+# We give some flexibility to go over 88, there are cases like long URLs or
+# code in documenation that have extra indentation. Black will still take care
+# of making everything that can be 88 wide, 88 wide.
+max-line-length = 100
+extend-ignore = [
+  "E203", # Whitespace before ':' (conflicts with black)
+  "W503", # Line break before binary operator (conflicts with black)
+]
+# pydoclint options
+style = "google"
+check-return-types = false
+check-yield-types = false
+arg-type-hints-in-docstring = false
+arg-type-hints-in-signature = true
+allow-init-docstring = true
+
 [tool.pylint.similarities]
 ignore-comments = ['yes']
 ignore-docstrings = ['yes']

cookiecutter/{{cookiecutter.github_repo_name}}/pyproject.toml

@@ -61,10 +61,12 @@ extra-lint-examples = [
   "pytest >= 7.3.0, < 8",
   "sybil >= 5.0.3, < 6",
 ]
-dev-docstrings = [
+dev-flake8 = [
+  "flake8 == 6.1.0",
+  "flake8-docstrings == 1.7.0",
+  "flake8-pyproject == 1.2.3",  # For reading the flake8 config from pyproject.toml
+  "pydoclint == 0.3.0",
   "pydocstyle == 6.3.0",
-  "darglint == 1.8.1",
-  "tomli == 2.0.1",      # Needed by pydocstyle to read pyproject.toml
 ]
 dev-formatting = ["black == 23.7.0", "isort == 5.12.0"]
 dev-mkdocs = [
@@ -94,7 +96,7 @@ dev-pytest = [
   "sybil == 5.0.3",        # Should be consistent with the extra-lint-examples dependency
 ]
 dev = [
-  "frequenz-repo-config[dev-mkdocs,dev-docstrings,dev-formatting,dev-mkdocs,dev-mypy,dev-noxfile,dev-pylint,dev-pytest]",
+  "frequenz-repo-config[dev-mkdocs,dev-flake8,dev-formatting,dev-mkdocs,dev-mypy,dev-noxfile,dev-pylint,dev-pytest]",
 ]
 
 [project.urls]
@@ -113,6 +115,23 @@ profile = "black"
 line_length = 88
 src_paths = ["benchmarks", "examples", "src", "tests"]
 
+[tool.flake8]
+# We give some flexibility to go over 88, there are cases like long URLs or
+# code in documenation that have extra indentation. Black will still take care
+# of making everything that can be 88 wide, 88 wide.
+max-line-length = 100
+extend-ignore = [
+  "E203", # Whitespace before ':' (conflicts with black)
+  "W503", # Line break before binary operator (conflicts with black)
+]
+# pydoclint options
+style = "google"
+check-return-types = false
+check-yield-types = false
+arg-type-hints-in-docstring = false
+arg-type-hints-in-signature = true
+allow-init-docstring = true
+
 [tool.pylint.similarities]
 ignore-comments = ['yes']
 ignore-docstrings = ['yes']

pyproject.toml

@@ -73,12 +73,12 @@
 
 The following optional dependencies are used and must be defined:
 
-- `dev-docstrings`: Dependencies to lint the documentation.
+- `dev-flake8`: Dependencies to do flake8 lint, including the documentation.
 
   At least these packages should be included:
 
   - `pydocstyle`: To check the docstrings' format.
-  - `darglint`: To check the docstrings' content.
+  - `pydoclint`: To check the docstrings' content.
 
 - `dev-formatting`: Dependencies to check the code's formatting.
 
@@ -130,7 +130,13 @@
 # ...
 
 [project.optional-dependencies]
-dev-docstrings = ["pydocstyle == 6.3.0", "darglint == 1.8.1"]
+dev-flake8 = [
+  "flake8 == 6.1.0",
+  "flake8-docstrings == 1.7.0",
+  "flake8-pyproject == 1.2.3",  # For reading the flake8 config from pyproject.toml
+  "pydoclint == 0.2.4",
+  "pydocstyle == 6.3.0",
+]
 dev-formatting = ["black == 23.3.0", "isort == 5.12.0"]
 dev-mkdocs = [
   "mike == 1.1.2",
@@ -157,7 +163,7 @@
   "pytest-mock == 3.10.0",
 ]
 dev = [
-  "my-package[dev-mkdocs,dev-docstrings,dev-formatting,dev-mypy,dev-nox,dev-pylint,dev-pytest]",
+  "my-package[dev-mkdocs,dev-flake8,dev-formatting,dev-mypy,dev-nox,dev-pylint,dev-pytest]",
 ]
 ```
 

src/frequenz/repo/config/__init__.py

@@ -29,18 +29,15 @@ class CommandsOptions:
     black: list[str] = _dataclasses.field(default_factory=lambda: [])
     """Command-line options for the `black` command."""
 
-    darglint: list[str] = _dataclasses.field(default_factory=lambda: [])
-    """Command-line options for the `darglint` command."""
+    flake8: list[str] = _dataclasses.field(default_factory=lambda: [])
+    """Command-line options for the `flake8` command."""
 
     isort: list[str] = _dataclasses.field(default_factory=lambda: [])
     """Command-line options for the `isort` command."""
 
     mypy: list[str] = _dataclasses.field(default_factory=lambda: [])
     """Command-line options for the `mypy` command."""
 
-    pydocstyle: list[str] = _dataclasses.field(default_factory=lambda: [])
-    """Command-line options for the `pydocstyle` command."""
-
     pylint: list[str] = _dataclasses.field(default_factory=lambda: [])
     """Command-line options for the `pylint` command."""
 
@@ -56,10 +53,9 @@ def copy(self) -> Self:
         return _dataclasses.replace(
             self,
             black=self.black.copy(),
-            darglint=self.darglint.copy(),
+            flake8=self.flake8.copy(),
             isort=self.isort.copy(),
             mypy=self.mypy.copy(),
-            pydocstyle=self.pydocstyle.copy(),
             pylint=self.pylint.copy(),
             pytest=self.pytest.copy(),
         )
@@ -239,7 +235,7 @@ def configure(
     # We need to make sure sessions are imported, otherwise they won't be visible to nox.
     if import_default_sessions:
         # pylint: disable=import-outside-toplevel,cyclic-import
-        from . import session as _
+        from . import session as _  # noqa: F401
 
     match conf:
         case Config():

src/frequenz/repo/config/nox/config.py

@@ -30,9 +30,7 @@
     black=[
         "--check",
     ],
-    darglint=[
-        "-v2",  # for verbose error messages.
-    ],
+    flake8=[],
     isort=[
         "--diff",
         "--check",
@@ -56,9 +54,9 @@
     opts=common_command_options.copy(),
     sessions=[
         "formatting",
+        "flake8",
         "mypy",
         "pylint",
-        "docstrings",
         "pytest_min",
         "pytest_max",
     ],