|
27 | 27 | # bookserver = { path = "../BookServer", develop = true } |
28 | 28 | # runestone = { path = "../RunestoneComponents", develop = true } |
29 | 29 | # |
30 | | -# ...in production mode; it does the opposite (changes ``[tool.poetry.dependencies]`` to ``[tool.no-poetry.dependencies]``) in development mode. This hides the modified section from Poetry, so the file now looks like an either/or project. |
31 | | -# |
32 | | -# #. Poetry doesn't install development dependencies in projects included through a `path dependency <https://python-poetry.org/docs/dependency-specification/#path-dependencies>`_. As a workaround, this script copies development dependencies from a project into an otherwise empty, auto-created "project", but puts them in the production dependencies section of this newly-created "project", so they will be installed. For example, the BookServer ``pyproject.toml`` contains: |
33 | | -# |
34 | | -# .. code-block:: text |
35 | | -# |
36 | | -# [tool.poetry.dev-dependencies] |
37 | | -# black = "~= 22.0" |
38 | | -# console-ctrl = "^0.1.0" |
39 | | -# ...many more, which are omitted for clarity... |
40 | | -# |
41 | | -# Poetry won't install these. Therefore, `make_dev_pyproject <make_dev_pyproject>` creates a "project" named bookserver-dev whose ``pyproject.toml`` contains a copy of the BookServer development dependencies, but placed in the production dependencies section of this ``bookserver-dev`` "project", so they will be installed. For example, the bookserver-dev ``pyproject.toml`` contains: |
42 | | -# |
43 | | -# .. code-block:: text |
44 | | -# |
45 | | -# [tool.poetry.dependencies] # <== CHANGED! |
46 | | -# black = "~= 22.0" |
47 | | -# console-ctrl = "^0.1.0" |
48 | | -# ...many more, which are omitted for clarity... |
49 | | -# |
50 | | -# This also means that the RunestoneServer ``pyproject.toml`` file must be manually edited to include a reference to this "project": |
51 | | -# |
52 | | -# .. code-block:: text |
53 | | -# |
54 | | -# [tool.poetry.dev-dependencies] |
55 | | -# bookserver = { path = "../BookServer", develop = true } |
56 | | -# bookserver-dev = { path = "../bookserver-dev", develop = true } # <== MANUALLY ADDED! |
57 | | -# |
58 | | -# The final result looks like this: |
59 | | -# |
60 | | -# .. image:: poetry_fix_diagram.png |
61 | | -# |
62 | | -# #. Poetry generates invalid package metadata for local path dependencies, so that running ``pip show click`` results in a bunch of exceptions. This program doesn't provide a fix for this bug. |
63 | | -# |
64 | | -# ...and that's how using Poetry makes dependency management easier... |
65 | | -# |
66 | | -# |
67 | | -# `Invalid package METADATA <https://github.com/python-poetry/poetry/issues/3148>`_ |
68 | | -# ===================================================================================== |
69 | | -# Per the issue linked in the title above, Poetry generates invalid package metadata for local path dependencies (tested on Poetry v1.1.14). For example, the last few lines of ``.venv/lib/python3.8/site-packages/runestone_poetry_project-0.1.0.dist-info/METADATA`` contain: |
70 | | -# |
71 | | -# .. code-block:: text |
72 | | -# |
73 | | -# Requires-Dist: pytz (>=2016.6.1) |
74 | | -# Requires-Dist: requests (>=2.10.0) |
75 | | -# Requires-Dist: rsmanage @ rsmanage |
76 | | -# Requires-Dist: runestone |
77 | | -# Requires-Dist: runestone-docker-tools @ docker |
78 | | -# Requires-Dist: six (>=1.10.0) |
79 | | -# Requires-Dist: sphinxcontrib-paverutils (>=1.17) |
80 | | -# Requires-Dist: stripe (>=2.0.0,<3.0.0) |
81 | | -# |
82 | | -# This causes an exception when running a command such as ``pip show click``: |
83 | | -# |
84 | | -# .. code-block:: text |
85 | | -# |
86 | | -# ERROR: Exception: |
87 | | -# Traceback (most recent call last): |
88 | | -# File "/srv/web2py/applications/runestone/.venv/lib/python3.8/site-packages/pip/_vendor/pkg_resources/__init__.py", line 3021, in _dep_map |
89 | | -# return self.__dep_map |
90 | | -# File "/srv/web2py/applications/runestone/.venv/lib/python3.8/site-packages/pip/_vendor/pkg_resources/__init__.py", line 2815, in __getattr__ |
91 | | -# raise AttributeError(attr) |
92 | | -# AttributeError: _DistInfoDistribution__dep_map |
93 | | -# |
94 | | -# ... along with a long traceback of other chained exceptions. |
95 | | -# |
96 | | -# Fixing the ``METADATA`` file to be: |
97 | | -# |
98 | | -# .. code-block:: text |
99 | | -# |
100 | | -# Requires-Dist: pytz (>=2016.6.1) |
101 | | -# Requires-Dist: requests (>=2.10.0) |
102 | | -# Requires-Dist: rsmanage @ file://rsmanage |
103 | | -# Requires-Dist: runestone |
104 | | -# Requires-Dist: runestone-docker-tools @ file://docker |
105 | | -# Requires-Dist: six (>=1.10.0) |
106 | | -# Requires-Dist: sphinxcontrib-paverutils (>=1.17) |
107 | | -# Requires-Dist: stripe (>=2.0.0,<3.0.0) |
108 | | -# |
109 | | -# ... along with a similar fix to the ``METADATA`` for ``bookserver_dev`` allows ``pip`` to run successfully. |
| 30 | +# ...in production mode; it does the opposite (changes ``[tool.poetry.dev-dependencies]`` to ``[tool.no-poetry.dev-dependencies]``) in development mode. This hides the modified section from Poetry, so the file now looks like an either/or project. |
110 | 31 | # |
111 | 32 | # |
112 | 33 | # TODO |
|
121 | 42 | # Standard library |
122 | 43 | # ---------------- |
123 | 44 | from pathlib import Path |
124 | | -import sys |
125 | | -from typing import Any, Dict, Set |
126 | 45 |
|
127 | 46 | # Third-party imports |
128 | 47 | # ------------------- |
129 | 48 | import click |
130 | | -import toml |
131 | 49 |
|
132 | 50 |
|
133 | 51 | # Local application imports |
134 | 52 | # ------------------------- |
135 | 53 | # None. |
136 | 54 | # |
137 | | -# Fix for ``dev-dependencies`` in subprojects |
138 | | -# =========================================== |
139 | | -# Given a main Poetry ``pyproject.toml``, these functions look for all subprojects included via path dependencies, creating additional subprojects named ``projectname-dev`` in which the subproject's dev-dependencies become dependencies in the newly-created subproject. This is a workaround for Poetry's inability to install the dev dependencies for a sub project included via a path requirement. To use this, in the main project, do something like: |
140 | | -# |
141 | | -# .. code-block:: TOML |
142 | | -# :linenos: |
143 | | -# |
144 | | -# [tool.poetry.dev-dependencies] |
145 | | -# sub = { path = "../sub", develop = true } |
146 | | -# sub-dev = { path = "../sub-dev", develop = true } |
147 | | -# |
148 | | -# Create a project clone where the original project's dev-dependencies are dependencies in the clone. |
149 | | -def create_dev_dependencies( |
150 | | - # The path to the project. |
151 | | - project_path: Path, |
152 | | -) -> None: |
153 | | - # Create a dev-only flavor. |
154 | | - d = toml.load(project_path / "pyproject.toml") |
155 | | - tp = d["tool"]["poetry"] |
156 | | - dd = "dev-dependencies" |
157 | | - # If there are no dev-dependencies, there's nothing to do. Otherwise, move them to dependencies. |
158 | | - if dd not in tp: |
159 | | - return |
160 | | - tp["dependencies"] = tp.pop(dd) |
161 | | - # Update the project name. |
162 | | - project_name = tp["name"] = tp["name"] + "-dev" |
163 | | - # We don't have a readme -- if it exists, Poetry will complain about the missing file it references. Remove it if it exists. |
164 | | - tp.pop("readme", None) |
165 | | - |
166 | | - # Put the output in a ``project_name-dev/`` directory. |
167 | | - dev = project_path.parent / project_name |
168 | | - print(f"Creating {dev}...") |
169 | | - dev.mkdir(exist_ok=True) |
170 | | - (dev / "pyproject.toml").write_text(toml.dumps(d)) |
171 | | - |
172 | | - # Create a minimal project to make Poetry happy. |
173 | | - project_name = project_name.replace("-", "_") |
174 | | - p = dev / project_name |
175 | | - p.mkdir(exist_ok=True) |
176 | | - (p / "__init__.py").write_text("") |
177 | | - |
178 | | - |
179 | | -def walk_dependencies( |
180 | | - # A dict of Poetry-specific values. |
181 | | - poetry_dict: Dict[str, Any], |
182 | | - # True to look at dependencies; False to look at dev-dependencies. |
183 | | - is_deps: bool, |
184 | | - # See `project_path`. |
185 | | - project_path: Path, |
186 | | - # See `walked_paths_set`. |
187 | | - walked_paths_set: Set[Path], |
188 | | - # See `poetry_paths_set`. |
189 | | - poetry_paths_set: Set[Path], |
190 | | -): |
191 | | - key = "dependencies" if is_deps else "dev-dependencies" |
192 | | - for dep in poetry_dict.get(key, {}).values(): |
193 | | - pth = dep.get("path", "") if isinstance(dep, dict) else None |
194 | | - if pth: |
195 | | - walk_pyproject(project_path / pth, walked_paths_set, poetry_paths_set) |
196 | | - |
197 | | - |
198 | | -# Given a ``pyproject.toml``, optionally create a dev dependencies project and walk all requirements with path dependencies. |
199 | | -def walk_pyproject( |
200 | | - # The path where a ``pyproject.toml`` exists. |
201 | | - project_path: Path, |
202 | | - # _`walked_paths_set`: a set of Paths already walked. |
203 | | - walked_paths_set: Set[Path], |
204 | | - # _`poetry_paths_set`: a set of Paths that contained a Poetry project. This is a strict subset of walked_paths_set_. |
205 | | - poetry_paths_set: Set[Path], |
206 | | - # True if this is the root ``pyproject.toml`` file -- no dev dependencies will be created for it. |
207 | | - is_root: bool = False, |
208 | | -): |
209 | | - project_path = project_path.resolve() |
210 | | - # Avoid cycles and unnecessary work. |
211 | | - if project_path in walked_paths_set: |
212 | | - return |
213 | | - walked_paths_set.add(project_path) |
214 | | - print(f"Examining {project_path} ...") |
215 | | - |
216 | | - # Process dependencies, if this is a Poetry project. |
217 | | - try: |
218 | | - d = toml.load(project_path / "pyproject.toml") |
219 | | - except FileNotFoundError: |
220 | | - return |
221 | | - poetry_paths_set.add(project_path) |
222 | | - tp = d["tool"]["poetry"] |
223 | | - # Search both the dependencies and dev dependencies in this project for path dependencies. |
224 | | - walk_dependencies(tp, True, project_path, walked_paths_set, poetry_paths_set) |
225 | | - walk_dependencies(tp, False, project_path, walked_paths_set, poetry_paths_set) |
226 | | - |
227 | | - # (Usually) process this file. |
228 | | - if not is_root: |
229 | | - create_dev_dependencies(project_path) |
230 | | - |
231 | | - |
232 | | -# .. _make_dev_pyproject: |
233 | | -# |
234 | | -# Core function: run the whole process on the ``pyproject.toml`` in the current directory. |
235 | | -def make_dev_pyproject(): |
236 | | - project_paths_set = set() |
237 | | - walk_pyproject(Path("."), set(), project_paths_set, True) |
238 | | - |
239 | | - # Check that we processed the BookServer and the RunestoneComponents. |
240 | | - found_bookserver = False |
241 | | - found_runestone_components = False |
242 | | - for path in project_paths_set: |
243 | | - name = path.name |
244 | | - found_bookserver |= name == "BookServer" |
245 | | - found_runestone_components |= name == "RunestoneComponents" |
246 | | - if not found_bookserver: |
247 | | - sys.exit("Error: did not process the BookServer Poetry project.") |
248 | | - if not found_runestone_components: |
249 | | - sys.exit("Error: did not process the RunestoneComponents Poetry project.") |
250 | | - |
251 | | - |
252 | 55 | # .. _rename_pyproject: |
253 | 56 | # |
254 | 57 | # Workaround for the main ``pyproject.toml`` |
@@ -306,12 +109,10 @@ def rewrite_pyproject(is_dev: bool) -> None: |
306 | 109 | ) |
307 | 110 | def main(no_dev: bool): |
308 | 111 | """ |
309 | | - This script works around Poetry bugs related to path dependencies. |
| 112 | + This script works around Poetry limitations to provide support of either/or dependencies. |
310 | 113 | """ |
311 | 114 | is_dev = not no_dev |
312 | 115 | rewrite_pyproject(is_dev) |
313 | | - if is_dev: |
314 | | - make_dev_pyproject() |
315 | 116 |
|
316 | 117 |
|
317 | 118 | if __name__ == "__main__": |
|
0 commit comments