Miscelaneous improvements (#16)

- Rename dev `nox` sessions to make them more explicit
- Rename optional dependencies to match `nox` sessions
- Add optional dependencies for actor, app and lib
- Move `py.typed` to `frequenz.repo.config`
- Use imports from the top-level package
- Make `flatten()` really return an iterator
- Auto-discover `pytest` paths
- Fix `pytest` invocation
- Improve general module documentation

Author: llucax

noxfile.py

@@ -3,10 +3,10 @@
 
 """Configuration file for nox."""
 
-from frequenz.repo import config
+from frequenz.repo.config import nox
 
 # Remove the pytest sessions because we don't have tests yet
-conf = config.nox.default.lib_config.copy()
-conf.sessions = [s for s in conf.sessions if not s.startswith("pytest")]
+config = nox.default.lib_config.copy()
+config.sessions = [s for s in config.sessions if not s.startswith("pytest")]
 
-config.nox.configure(conf)
+nox.configure(config)

pyproject.toml

@@ -37,35 +37,40 @@ name = "Frequenz Energy-as-a-Service GmbH"
 email = "[email protected]"
 
 [project.optional-dependencies]
+actor = []
 api = [
   "grpcio-tools >= 1.47.0, < 2",
   "mypy-protobuf >= 3.0.0, < 4",
   "setuptools >= 67.6.0, < 68",
 ]
-docs-gen = [
+app = []
+lib = []
+dev-docs-gen = [
   "mike == 1.1.2",
   "mkdocs-gen-files == 0.4.0",
   "mkdocs-literate-nav == 0.4.0",
   "mkdocs-material == 9.1.4",
   "mkdocs-section-index == 0.3.5",
   "mkdocstrings[python] == 0.20.0",
 ]
-docs-lint = ["pydocstyle == 6.3.0", "darglint == 1.8.1"]
-format = ["black == 23.3.0", "isort == 5.12.0"]
-mypy = [
+dev-docstrings = ["pydocstyle == 6.3.0", "darglint == 1.8.1"]
+dev-formatting = ["black == 23.3.0", "isort == 5.12.0"]
+dev-mypy = [
   "mypy == 1.1.1",
   "types-setuptools >= 67.6.0, < 68", # Should match the global dependency
   # For checking the docs/ script, and tests
   "frequenz-repo-config[docs-gen,pytest]",
 ]
-pylint = [
+dev-pylint = [
   "pylint == 2.17.1",
   "pylint-google-style-guide-imports-enforcing == 1.3.0",
   # For checking the docs/ script, and tests
   "frequenz-repo-config[docs-gen,pytest]",
 ]
-pytest = ["pytest == 7.2.2"]
-dev = ["frequenz-repo-config[docs-gen,docs-lint,format,pytest,mypy,pylint]"]
+dev-pytest = ["pytest == 7.2.2"]
+dev = [
+  "frequenz-repo-config[dev-docs-gen,dev-docstrings,dev-formatting,dev-pytest,dev-mypy,dev-pylint]",
+]
 
 [project.urls]
 Changelog = "https://github.com/frequenz-floss/frequenz-repo-config-python/releases"

src/frequenz/repo/config/__init__.py

@@ -14,24 +14,11 @@
 
 ## `nox`
 
+### Writing the `noxfile.py`
+
 Projects wanting to use `nox` to run lint checkers and other utilities can use
 the [`frequenz.repo.config.nox`][] package.
 
-Make sure to add this package as `dev` dependency to your project's
-`pyproject.tom` file, for example:
-
-```toml
-[project.optional-dependencies]
-nox = [
-    "nox == 2022.11.21",
-    "frequenz-repo-config[lib] == 0.1.0",
-]
-```
-
-Please note the `lib` optional dependency. Make sure you specify the correct
-one based on the type of your project (`api`, `actor`, `app`, `lib`). Also make
-sure to adjust the versions.
-
 When writing the `noxfile.py` you should import the `nox` module from this
 package and use the [`frequenz.repo.config.nox.configure`][] function,
 which will configure all nox sessions.
@@ -40,9 +27,9 @@
 in the [`frequenz.repo.config.nox.default`][] module. For example:
 
 ```python
-from frequenz.repo import config
+from frequenz.repo.config import nox
 
-config.nox.configure(config.nox.default.lib_config)
+nox.configure(nox.default.lib_config)
 ```
 
 Again, make sure to pick the correct default configuration based on the type of
@@ -53,11 +40,11 @@
 [`copy()`][frequenz.repo.config.nox.config.Config.copy] method:
 
 ```python
-from frequenz.repo import config
+from frequenz.repo.config import nox
 
-conf = config.nox.default.lib_config.copy()
-conf.opts.black.append("--diff")
-config.nox.configure(conf)
+config = nox.default.lib_config.copy()
+config.opts.black.append("--diff")
+nox.configure(config)
 ```
 
 If you need further customization or to define new sessions, you can use the
@@ -73,6 +60,93 @@
 
 - [`frequenz.repo.config.nox.util`][]: General purpose utility functions.
 
+### `pyproject.toml` configuration
+
+All sessions configured by this package expect the `pyproject.toml` file to
+define specific *dev* dependencies that will be used by the different `nox`
+sessions.
+
+The following optional dependencies are used and must be defined:
+
+- `dev-docstrings`: Dependencies to lint the documentation.
+
+  At least these packages should be included:
+
+  - `pydocstyle`: To check the docstrings' format.
+  - `darglint`: To check the docstrings' content.
+
+- `dev-formatting`: Dependencies to check the code's formatting.
+
+  At least these packages should be included:
+
+  - `black`: To check the code's formatting.
+  - `isort`: To check the imports' formatting.
+
+- `dev-mypy`: Dependencies to run `mypy` to check the code's type annotations.
+
+  At least these packages should be included:
+
+  - `mypy`: To check the code's type annotations.
+
+- `dev-pylint`: Dependencies to run `pylint` to lint the code.
+
+  At least these packages should be included:
+
+  - `pylint`: To lint the code.
+
+- `dev-pytest`: Dependencies to run the tests using `pytest`.
+
+  At least these packages should be included:
+
+  - `pytest`: To run the tests.
+
+For some of these you should install too any other dependencies that are used
+by the project. For example, if the project uses `pytest-asyncio`, you should
+include it in the `dev-pytest` optional dependency.
+
+It is also recommended, but not required, to provide a global `dev` optional
+dependency that includes all the other optional dependencies, so users can
+install all the dependencies needed while developing the project without having
+to run `nox`, which might be a bit slow if you want to do quick iterations.
+
+```console
+$ pip install -e .[dev]
+...
+$ pytest
+...
+```
+
+Here is a sample `pyproject.toml` file that defines all the optional
+dependencies:
+
+```toml
+[project]
+name = "my-package"
+# ...
+
+[project.optional-dependencies]
+dev-docstrings = ["pydocstyle == 6.3.0", "darglint == 1.8.1"]
+dev-formatting = ["black == 23.3.0", "isort == 5.12.0"]
+dev-mypy = [
+  "mypy == 1.1.1",
+  # For checking tests
+  "my-package[dev-pytest]",
+]
+dev-pylint = [
+  "pylint == 2.17.1",
+  "pylint-google-style-guide-imports-enforcing == 1.3.0",
+  # For checking tests
+  "my-package[dev-pytest]",
+]
+dev-pytest = [
+  "pytest == 7.2.2",
+  "pytest-asyncio == 0.21.0",
+  "pytest-mock == 3.10.0",
+]
+dev = [
+  "my-package[dev-docstrings,dev-formatting,dev-mypy,dev-nox,dev-pylint,dev-pytest]",
+]
+```
 
 # APIs
 

src/frequenz/repo/config/nox/config.py

@@ -84,6 +84,15 @@ class Config:
     tools invoked by the sessions.
     """
 
+    def __post_init__(self) -> None:
+        """Initialize the configuration object.
+
+        This will add extra paths discovered in config files and other sources.
+        """
+        for path in util.discover_paths():
+            if path not in self.extra_paths:
+                self.extra_paths.append(path)
+
     def copy(self, /) -> Self:
         """Create a new object as a copy of self.
 

src/frequenz/repo/config/nox/default.py

@@ -27,7 +27,10 @@
 method.
 """
 
+import dataclasses
+
 from . import config as _config
+from . import util
 
 common_command_options: _config.CommandsOptions = _config.CommandsOptions(
     black=[
@@ -86,8 +89,16 @@
 api_command_options: _config.CommandsOptions = common_command_options.copy()
 """Default command-line options for APIs."""
 
-api_config: _config.Config = common_config.copy()
-"""Default configuration for APIs."""
+api_config: _config.Config = dataclasses.replace(
+    common_config,
+    source_paths=list(util.replace(common_config.source_paths, {"src": "py"})),
+    extra_paths=list(util.replace(common_config.extra_paths, {"tests": "pytests"})),
+)
+"""Default configuration for APIs.
+
+Same as `common_config`, but with `source_paths` replacing `"src"` with `"py"`
+and `extra_paths` replacing `"tests"` with `"pytests"`.
+"""
 
 actor_command_options: _config.CommandsOptions = common_command_options.copy()
 """Default command-line options for actors."""

src/frequenz/repo/config/nox/session.py

@@ -40,7 +40,7 @@ def formatting(session: nox.Session, install_deps: bool = True) -> None:
         install_deps: True if dependencies should be installed.
     """
     if install_deps:
-        session.install("-e", ".[format]")
+        session.install("-e", ".[dev-formatting]")
 
     conf = config.get()
     session.run("black", *conf.opts.black, *conf.path_args(session))
@@ -58,7 +58,7 @@ def mypy(session: nox.Session, install_deps: bool = True) -> None:
     if install_deps:
         # install the package itself as editable, so that it is possible to do
         # fast local tests with `nox -R -e mypy`.
-        session.install("-e", ".[mypy]")
+        session.install("-e", ".[dev-mypy]")
 
     conf = config.get()
     pkg_args = util.flatten(("-p", p) for p in conf.package_args(session))
@@ -76,7 +76,7 @@ def pylint(session: nox.Session, install_deps: bool = True) -> None:
     if install_deps:
         # install the package itself as editable, so that it is possible to do
         # fast local tests with `nox -R -e pylint`.
-        session.install("-e", ".[pylint]")
+        session.install("-e", ".[dev-pylint]")
 
     conf = config.get()
     session.run("pylint", *conf.opts.pylint, *conf.path_args(session))
@@ -91,7 +91,7 @@ def docstrings(session: nox.Session, install_deps: bool = True) -> None:
         install_deps: True if dependencies should be installed.
     """
     if install_deps:
-        session.install("-e", ".[docs-lint]")
+        session.install("-e", ".[dev-docstrings]")
 
     conf = config.get()
     session.run("pydocstyle", *conf.opts.pydocstyle, *conf.path_args(session))
@@ -120,7 +120,7 @@ def pytest_max(session: nox.Session, install_deps: bool = True) -> None:
     if install_deps:
         # install the package itself as editable, so that it is possible to do
         # fast local tests with `nox -R -e pytest_max`.
-        session.install("-e", ".[pytest]")
+        session.install("-e", ".[dev-pytest]")
 
     _pytest_impl(session, "max")
 
@@ -136,7 +136,7 @@ def pytest_min(session: nox.Session, install_deps: bool = True) -> None:
     if install_deps:
         # install the package itself as editable, so that it is possible to do
         # fast local tests with `nox -R -e pytest_min`.
-        session.install("-e", ".[pytest]", *util.min_dependencies())
+        session.install("-e", ".[dev-pytest]", *util.min_dependencies())
 
     _pytest_impl(session, "min")
 
@@ -145,7 +145,7 @@ def _pytest_impl(
     session: nox.Session, max_or_min_deps: str  # pylint: disable=unused-argument
 ) -> None:
     conf = config.get()
-    session.run("pytest", *conf.opts.pytest, *conf.path_args(session))
+    session.run("pytest", *conf.opts.pytest, *session.posargs)
 
     # pylint: disable=fixme
     # TODO: Implement coverage reporting, we need to research this a bit and it

src/frequenz/repo/config/nox/util.py

@@ -10,7 +10,7 @@
 
 import pathlib
 import tomllib
-from collections.abc import Iterable
+from collections.abc import Iterable, Mapping, Set
 from typing import TypeVar
 
 _T = TypeVar("_T")
@@ -28,7 +28,28 @@ def flatten(iterables: Iterable[Iterable[_T]], /) -> Iterable[_T]:
     Example:
         >>> assert list(flatten([(1, 2), (3, 4)]) == [1, 2, 3, 4]
     """
-    return [item for sublist in iterables for item in sublist]
+    return (item for sublist in iterables for item in sublist)
+
+
+def replace(iterable: Iterable[_T], replacements: Mapping[_T, _T], /) -> Iterable[_T]:
+    """Replace elements in an iterable.
+
+    Args:
+        iterable: The iterable to replace elements in.
+        old: The elements to replace.
+        new: The elements to replace with.
+
+    Returns:
+        An iterable with the elements in `iterable` replaced.
+
+    Example:
+        >>> assert list(replace([1, 2, 3], old={1, 2}, new={4, 5})) == [4, 5, 3]
+    """
+    for item in iterable:
+        if item in replacements:
+            yield replacements[item]
+        else:
+            yield item
 
 
 def existing_paths(paths: Iterable[str], /) -> Iterable[pathlib.Path]:
@@ -170,3 +191,30 @@ def min_dependencies() -> list[str]:
         else:
             raise RuntimeError(f"Minimum requirement is not set: {dep}")
     return min_deps
+
+
+def discover_paths() -> list[str]:
+    """Discover paths to check.
+
+    Discover the paths to check by looking into different sources, like the
+    `pyproject.toml` file.
+
+    Currently the following paths are discovered:
+
+    - The `testpaths` option in the `tools.pytest.ini_options` section of
+      `pyproject.toml`.
+
+    Returns:
+        The discovered paths to check.
+    """
+    with open("pyproject.toml", "rb") as toml_file:
+        data = tomllib.load(toml_file)
+
+    testpaths = (
+        data.get("tool", {})
+        .get("pytest", {})
+        .get("ini_options", {})
+        .get("testpaths", [])
+    )
+
+    return testpaths