Replace darglint with pydoclint via flake8

The `darglint` project is not maintained anymore, and even when there is
a fork that is trying to keep it alive, it is still extremely slow.

The new project `pydoclint` was created recently but advancing rapidly,
and already in better shape than `darglint` and way faster (0.2s vs 20s
for the SDK, 100x improvement).

To be able to add ignore comments in the code, `pydoclint` needs to be
run via `flake8`. Because we need to run `flake8` already, and
`pydocstyle` also can run via `flake8`, we are merging both in one
`flake8` call, so we only need to read the files once.

As a side effect, we are also enabling other base `flake8` checks. There
is probably some overlap with `pylint`, but `flake8` is very fast
anyway, so it shouldn't be noticed (for the SDK, `flake8` basic checks
+ `pydocstyle` + `pydoclint` runs in 0.6s and `pylint` alone runs in
15s). At some point we might want to disable the duplicated checks in
`pylint` to see if it speeds up `pylint`.

This commit also renames the `dev-docstrings` optional dependency to
`dev-flake8` because now is not checking docstrings exclusively.

Signed-off-by: Leandro Lucarella <[email protected]>

Author: llucax

.darglint

@@ -1,5 +1,4 @@
 exclude .cookiecutter-replay.json
-exclude .darglint
 exclude .editorconfig
 exclude .gitignore
 exclude CODEOWNERS

MANIFEST.in

@@ -1,5 +1,4 @@
 exclude .cookiecutter-replay.json
-exclude .darglint
 exclude .editorconfig
 exclude .gitignore
 {%- if cookiecutter.type == "api" %}

cookiecutter/{{cookiecutter.github_repo_name}}/MANIFEST.in

@@ -60,9 +60,12 @@ email = "{{cookiecutter.author_email}}"
 
 # TODO(cookiecutter): Remove and add more optional dependencies if appropriate
 [project.optional-dependencies]
-dev-docstrings = [
+dev-flake8 = [
+  "flake8 == 6.1.0",
+  "flake8-docstrings == 1.7.0",
+  "flake8-pyproject == 1.2.3",  # For reading the flake8 config from pyproject.toml
+  "pydoclint == 0.3.0",
   "pydocstyle == 6.3.0",
-  "darglint == 1.8.1",
   "tomli == 2.0.1",      # Needed by pydocstyle to read pyproject.toml
 ]
 dev-formatting = ["black == 23.7.0", "isort == 5.12.0"]
@@ -99,7 +102,7 @@ dev-pytest = [
 {%- endif %}
 ]
 dev = [
-  "{{cookiecutter.pypi_package_name}}[dev-mkdocs,dev-docstrings,dev-formatting,dev-mkdocs,dev-mypy,dev-noxfile,dev-pylint,dev-pytest]",
+  "{{cookiecutter.pypi_package_name}}[dev-mkdocs,dev-flake8,dev-formatting,dev-mkdocs,dev-mypy,dev-noxfile,dev-pylint,dev-pytest]",
 ]
 
 [project.urls]
@@ -121,6 +124,23 @@ line_length = 88
 {#- We don't include "py" here for API because we don't want to check generated files #}
 src_paths = ["benchmarks", "examples", "src", "tests"]
 
+[tool.flake8]
+# We give some flexibility to go over 88, there are cases like long URLs or
+# code in documenation that have extra indentation. Black will still take care
+# of making everything that can be 88 wide, 88 wide.
+max-line-length = 100
+extend-ignore = [
+  "E203", # Whitespace before ':' (conflicts with black)
+  "W503", # Line break before binary operator (conflicts with black)
+]
+# pydoclint options
+style = "google"
+check-return-types = false
+check-yield-types = false
+arg-type-hints-in-docstring = false
+arg-type-hints-in-signature = true
+allow-init-docstring = true
+
 [tool.pylint.similarities]
 ignore-comments = ['yes']
 ignore-docstrings = ['yes']

cookiecutter/{{cookiecutter.github_repo_name}}/pyproject.toml

@@ -61,9 +61,12 @@ extra-lint-examples = [
   "pytest >= 7.3.0, < 8",
   "sybil >= 5.0.3, < 6",
 ]
-dev-docstrings = [
+dev-flake8 = [
+  "flake8 == 6.1.0",
+  "flake8-docstrings == 1.7.0",
+  "flake8-pyproject == 1.2.3",  # For reading the flake8 config from pyproject.toml
+  "pydoclint == 0.3.0",
   "pydocstyle == 6.3.0",
-  "darglint == 1.8.1",
   "tomli == 2.0.1",      # Needed by pydocstyle to read pyproject.toml
 ]
 dev-formatting = ["black == 23.7.0", "isort == 5.12.0"]
@@ -94,7 +97,7 @@ dev-pytest = [
   "sybil == 5.0.3",        # Should be consistent with the extra-lint-examples dependency
 ]
 dev = [
-  "frequenz-repo-config[dev-mkdocs,dev-docstrings,dev-formatting,dev-mkdocs,dev-mypy,dev-noxfile,dev-pylint,dev-pytest]",
+  "frequenz-repo-config[dev-mkdocs,dev-flake8,dev-formatting,dev-mkdocs,dev-mypy,dev-noxfile,dev-pylint,dev-pytest]",
 ]
 
 [project.urls]
@@ -113,6 +116,23 @@ profile = "black"
 line_length = 88
 src_paths = ["benchmarks", "examples", "src", "tests"]
 
+[tool.flake8]
+# We give some flexibility to go over 88, there are cases like long URLs or
+# code in documenation that have extra indentation. Black will still take care
+# of making everything that can be 88 wide, 88 wide.
+max-line-length = 100
+extend-ignore = [
+  "E203", # Whitespace before ':' (conflicts with black)
+  "W503", # Line break before binary operator (conflicts with black)
+]
+# pydoclint options
+style = "google"
+check-return-types = false
+check-yield-types = false
+arg-type-hints-in-docstring = false
+arg-type-hints-in-signature = true
+allow-init-docstring = true
+
 [tool.pylint.similarities]
 ignore-comments = ['yes']
 ignore-docstrings = ['yes']

pyproject.toml

@@ -73,12 +73,12 @@
 
 The following optional dependencies are used and must be defined:
 
-- `dev-docstrings`: Dependencies to lint the documentation.
+- `dev-flake8`: Dependencies to do flake8 lint, including the documentation.
 
   At least these packages should be included:
 
   - `pydocstyle`: To check the docstrings' format.
-  - `darglint`: To check the docstrings' content.
+  - `pydoclint`: To check the docstrings' content.
 
 - `dev-formatting`: Dependencies to check the code's formatting.
 
@@ -130,7 +130,13 @@
 # ...
 
 [project.optional-dependencies]
-dev-docstrings = ["pydocstyle == 6.3.0", "darglint == 1.8.1"]
+dev-flake8 = [
+  "flake8 == 6.1.0",
+  "flake8-docstrings == 1.7.0",
+  "flake8-pyproject == 1.2.3",  # For reading the flake8 config from pyproject.toml
+  "pydoclint == 0.2.4",
+  "pydocstyle == 6.3.0",
+]
 dev-formatting = ["black == 23.3.0", "isort == 5.12.0"]
 dev-mkdocs = [
   "mike == 1.1.2",
@@ -157,7 +163,7 @@
   "pytest-mock == 3.10.0",
 ]
 dev = [
-  "my-package[dev-mkdocs,dev-docstrings,dev-formatting,dev-mypy,dev-nox,dev-pylint,dev-pytest]",
+  "my-package[dev-mkdocs,dev-flake8,dev-formatting,dev-mypy,dev-nox,dev-pylint,dev-pytest]",
 ]
 ```
 

src/frequenz/repo/config/__init__.py

@@ -29,18 +29,15 @@ class CommandsOptions:
     black: list[str] = _dataclasses.field(default_factory=lambda: [])
     """Command-line options for the `black` command."""
 
-    darglint: list[str] = _dataclasses.field(default_factory=lambda: [])
-    """Command-line options for the `darglint` command."""
+    flake8: list[str] = _dataclasses.field(default_factory=lambda: [])
+    """Command-line options for the `flake8` command."""
 
     isort: list[str] = _dataclasses.field(default_factory=lambda: [])
     """Command-line options for the `isort` command."""
 
     mypy: list[str] = _dataclasses.field(default_factory=lambda: [])
     """Command-line options for the `mypy` command."""
 
-    pydocstyle: list[str] = _dataclasses.field(default_factory=lambda: [])
-    """Command-line options for the `pydocstyle` command."""
-
     pylint: list[str] = _dataclasses.field(default_factory=lambda: [])
     """Command-line options for the `pylint` command."""
 
@@ -56,10 +53,9 @@ def copy(self) -> Self:
         return _dataclasses.replace(
             self,
             black=self.black.copy(),
-            darglint=self.darglint.copy(),
+            flake8=self.flake8.copy(),
             isort=self.isort.copy(),
             mypy=self.mypy.copy(),
-            pydocstyle=self.pydocstyle.copy(),
             pylint=self.pylint.copy(),
             pytest=self.pytest.copy(),
         )

src/frequenz/repo/config/nox/config.py

@@ -30,9 +30,7 @@
     black=[
         "--check",
     ],
-    darglint=[
-        "-v2",  # for verbose error messages.
-    ],
+    flake8=[],
     isort=[
         "--diff",
         "--check",
@@ -56,9 +54,9 @@
     opts=common_command_options.copy(),
     sessions=[
         "formatting",
+        "flake8",
         "mypy",
         "pylint",
-        "docstrings",
         "pytest_min",
         "pytest_max",
     ],

src/frequenz/repo/config/nox/default.py

@@ -28,7 +28,7 @@ def ci_checks_max(session: nox.Session) -> None:
     formatting(session, False)
     mypy(session, False)
     pylint(session, False)
-    docstrings(session, False)
+    flake8(session, False)
     pytest_max(session, False)
 
 
@@ -84,30 +84,18 @@ def pylint(session: nox.Session, install_deps: bool = True) -> None:
 
 
 @nox.session
-def docstrings(session: nox.Session, install_deps: bool = True) -> None:
-    """Check docstring tone with pydocstyle and param descriptions with darglint.
+def flake8(session: nox.Session, install_deps: bool = True) -> None:
+    """Check for common errors and in particular documentation format and style.
 
     Args:
         session: the nox session.
         install_deps: True if dependencies should be installed.
     """
     if install_deps:
-        session.install("-e", ".[dev-docstrings]")
+        session.install("-e", ".[dev-flake8]")
 
     conf = _config.get()
-    session.run("pydocstyle", *conf.opts.pydocstyle, *conf.path_args(session))
-
-    # Darglint checks that function argument and return values are documented.
-    # This is needed only for the `src` dir, so we exclude the other top level
-    # dirs that contain code, unless some paths were specified by argument, in
-    # which case we use those untouched.
-    darglint_paths = session.posargs or filter(
-        # pylint: disable=fixme
-        # TODO: Make these exclusions configurable
-        lambda path: not (path.startswith("tests") or path.startswith("benchmarks")),
-        conf.path_args(session),
-    )
-    session.run("darglint", *conf.opts.darglint, *darglint_paths)
+    session.run("flake8", *conf.opts.flake8, *conf.path_args(session))
 
 
 @nox.session

src/frequenz/repo/config/nox/session.py

@@ -1,5 +1,4 @@
 exclude .cookiecutter-replay.json
-exclude .darglint
 exclude .editorconfig
 exclude .gitignore
 exclude CODEOWNERS