From 17b9cac9c365dfd0342ff623911b6e4be89ac95c Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Tue, 27 Feb 2024 22:32:06 +0100 Subject: [PATCH 1/4] ENH: do not include uncommitted changes in the sdist Including uncommitted changes in the sdist was implemented in #58 after the discussion in #53. However, the current behavior for which uncommitted changes to files under version control are included but not other files, is a hard to justify surprising half measure. After careful analysis, it has been determined that none of the use cases for this feature is still valid. Removing it makes the implementation easier, and the behavior less surprising and easier to document and explain. While at it, modernize the associated test. --- mesonpy/__init__.py | 87 ++++++++++++------------- tests/packages/long-path/meson.build | 5 ++ tests/packages/long-path/pyproject.toml | 11 ++++ tests/test_sdist.py | 45 ++++++++----- 4 files changed, 85 insertions(+), 63 deletions(-) create mode 100644 tests/packages/long-path/meson.build create mode 100644 tests/packages/long-path/pyproject.toml diff --git a/mesonpy/__init__.py b/mesonpy/__init__.py index 0455bfd19..1f26e9db2 100644 --- a/mesonpy/__init__.py +++ b/mesonpy/__init__.py @@ -865,64 +865,57 @@ def _meson_version(self) -> str: def sdist(self, directory: Path) -> pathlib.Path: """Generates a sdist (source distribution) in the specified directory.""" - # generate meson dist file + # Generate meson dist file. self._run(self._meson + ['dist', '--allow-dirty', '--no-tests', '--formats', 'gztar', *self._meson_args['dist']]) - # move meson dist file to output path dist_name = f'{self._metadata.distribution_name}-{self._metadata.version}' meson_dist_name = f'{self._meson_name}-{self._meson_version}' meson_dist_path = pathlib.Path(self._build_dir, 'meson-dist', f'{meson_dist_name}.tar.gz') - sdist = pathlib.Path(directory, f'{dist_name}.tar.gz') + sdist_path = pathlib.Path(directory, f'{dist_name}.tar.gz') - with tarfile.open(meson_dist_path, 'r:gz') as meson_dist, mesonpy._util.create_targz(sdist) as tar: + with tarfile.open(meson_dist_path, 'r:gz') as meson_dist, mesonpy._util.create_targz(sdist_path) as sdist: for member in meson_dist.getmembers(): - # calculate the file path in the source directory - assert member.name, member.name - member_parts = member.name.split('/') - if len(member_parts) <= 1: - continue - path = self._source_dir.joinpath(*member_parts[1:]) - - if not path.exists() and member.isfile(): - # File doesn't exists on the source directory but exists on - # the Meson dist, so it is generated file, which we need to - # include. - # See https://mesonbuild.com/Reference-manual_builtin_meson.html#mesonadd_dist_script - - # MESON_DIST_ROOT could have a different base name - # than the actual sdist basename, so we need to rename here + if member.isfile(): file = meson_dist.extractfile(member.name) - member.name = str(pathlib.Path(dist_name, *member_parts[1:]).as_posix()) - tar.addfile(member, file) - continue - - if not path.is_file(): - continue - info = tarfile.TarInfo(member.name) - file_stat = os.stat(path) - info.mtime = member.mtime - info.size = file_stat.st_size - info.mode = int(oct(file_stat.st_mode)[-3:], 8) - - # rewrite the path if necessary, to match the sdist distribution name - if dist_name != meson_dist_name: - info.name = pathlib.Path( - dist_name, - path.relative_to(self._source_dir) - ).as_posix() - - with path.open('rb') as f: - tar.addfile(info, fileobj=f) - - # add PKG-INFO to dist file to make it a sdist - pkginfo_info = tarfile.TarInfo(f'{dist_name}/PKG-INFO') - pkginfo_info.mtime = time.time() # type: ignore[assignment] + # Reset pax extended header. The tar archive member may be + # using pax headers to store some file metadata. The pax + # headers are not reset when the metadata is modified and + # they take precedence when the member is deserialized. + # This is relevant because when rewriting the member name, + # the length of the path may shrink from being more than + # 100 characters (requiring the path to be stored in the + # pax headers) to being less than 100 characters. When this + # happens, the tar archive member is serialized with the + # shorter name in the regular header and the longer one in + # the extended pax header. The archives handled here are + # not expected to use extended pax headers other than for + # the ones required to encode file metadata. The easiest + # solution is to reset the pax extended headers. + member.pax_headers = {} + + # Rewrite the path to match the sdist distribution name. + stem = member.name.split('/', 1)[1] + member.name = '/'.join((dist_name, stem)) + + # Reset owner and group to root:root. This mimics what + # 'git archive' does and makes the sdist reproducible upon + # being built by different users. + member.uname = member.gname = 'root' + member.uid = member.gid = 0 + + sdist.addfile(member, file) + + # Add 'PKG-INFO'. + member = tarfile.TarInfo(f'{dist_name}/PKG-INFO') + member.uid = member.gid = 0 + member.uname = member.gname = 'root' + member.mtime = time.time() metadata = bytes(self._metadata.as_rfc822()) - pkginfo_info.size = len(metadata) - tar.addfile(pkginfo_info, fileobj=io.BytesIO(metadata)) + member.size = len(metadata) + sdist.addfile(member, io.BytesIO(metadata)) - return sdist + return sdist_path def wheel(self, directory: Path) -> pathlib.Path: """Generates a wheel in the specified directory.""" diff --git a/tests/packages/long-path/meson.build b/tests/packages/long-path/meson.build new file mode 100644 index 000000000..272d7fe0f --- /dev/null +++ b/tests/packages/long-path/meson.build @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2024 The meson-python developers +# +# SPDX-License-Identifier: MIT + +project('very-long-project-name-that-makes-the-paths-within-the-sdist-exceed-100-characters-xxxxxxxxxxxxxxxxx', version: '1.0.0') diff --git a/tests/packages/long-path/pyproject.toml b/tests/packages/long-path/pyproject.toml new file mode 100644 index 000000000..70791dd42 --- /dev/null +++ b/tests/packages/long-path/pyproject.toml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2021 The meson-python developers +# +# SPDX-License-Identifier: MIT + +[build-system] +build-backend = 'mesonpy' +requires = ['meson-python'] + +[project] +name = 'long-path' +dynamic = ['version'] diff --git a/tests/test_sdist.py b/tests/test_sdist.py index fb698b53d..0be265dd0 100644 --- a/tests/test_sdist.py +++ b/tests/test_sdist.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: MIT import os +import pathlib import re import stat import sys @@ -122,37 +123,35 @@ def test_contents_subdirs(sdist_subdirs): def test_contents_unstaged(package_pure, tmp_path): - new_data = textwrap.dedent(''' - def bar(): - return 'foo' + new = textwrap.dedent(''' + def bar(): + return 'foo' ''').strip() - with open('pure.py', 'r') as f: - old_data = f.read() - - try: - with in_git_repo_context(): - with open('pure.py', 'w') as f, open('crap', 'x'): - f.write(new_data) + old = pathlib.Path('pure.py').read_text() + with in_git_repo_context(): + try: + pathlib.Path('pure.py').write_text(new) + pathlib.Path('other.py').touch() sdist_path = mesonpy.build_sdist(os.fspath(tmp_path)) - finally: - with open('pure.py', 'w') as f: - f.write(old_data) - os.unlink('crap') + finally: + pathlib.Path('pure.py').write_text(old) + pathlib.Path('other.py').unlink() with tarfile.open(tmp_path / sdist_path, 'r:gz') as sdist: names = {member.name for member in sdist.getmembers()} mtimes = {member.mtime for member in sdist.getmembers()} - read_data = sdist.extractfile('pure-1.0.0/pure.py').read().replace(b'\r\n', b'\n') + data = sdist.extractfile('pure-1.0.0/pure.py').read().replace(b'\r\n', b'\n') + # Verify that uncommitted changes are not included in the sdist. assert names == { 'pure-1.0.0/PKG-INFO', 'pure-1.0.0/meson.build', 'pure-1.0.0/pure.py', 'pure-1.0.0/pyproject.toml', } - assert read_data == new_data.encode() + assert data == old.encode() # All the archive members have a valid mtime. assert 0 not in mtimes @@ -192,3 +191,17 @@ def test_generated_files(sdist_generated_files): # All the archive members have a valid mtime. assert 0 not in mtimes + + +def test_long_path(sdist_long_path): + # See https://github.com/mesonbuild/meson-python/pull/587#pullrequestreview-2020891328 + # and https://github.com/mesonbuild/meson-python/pull/587#issuecomment-2075973593 + + with tarfile.open(sdist_long_path, 'r:gz') as sdist: + names = {member.name for member in sdist.getmembers()} + + assert names == { + 'long_path-1.0.0/PKG-INFO', + 'long_path-1.0.0/meson.build', + 'long_path-1.0.0/pyproject.toml' + } From e0f4385bad157a0e53b37e7ee8a438caa884c479 Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Tue, 23 Apr 2024 11:08:10 +0200 Subject: [PATCH 2/4] ENH: make sdist archives reproducible --- mesonpy/__init__.py | 23 +++++++++++++++++++++-- mesonpy/_util.py | 6 +++++- tests/test_sdist.py | 13 +++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/mesonpy/__init__.py b/mesonpy/__init__.py index 1f26e9db2..83653c2ce 100644 --- a/mesonpy/__init__.py +++ b/mesonpy/__init__.py @@ -31,7 +31,6 @@ import tarfile import tempfile import textwrap -import time import typing import warnings @@ -872,6 +871,7 @@ def sdist(self, directory: Path) -> pathlib.Path: meson_dist_name = f'{self._meson_name}-{self._meson_version}' meson_dist_path = pathlib.Path(self._build_dir, 'meson-dist', f'{meson_dist_name}.tar.gz') sdist_path = pathlib.Path(directory, f'{dist_name}.tar.gz') + pyproject_toml_mtime = 0 with tarfile.open(meson_dist_path, 'r:gz') as meson_dist, mesonpy._util.create_targz(sdist_path) as sdist: for member in meson_dist.getmembers(): @@ -898,6 +898,9 @@ def sdist(self, directory: Path) -> pathlib.Path: stem = member.name.split('/', 1)[1] member.name = '/'.join((dist_name, stem)) + if stem == 'pyproject.toml': + pyproject_toml_mtime = member.mtime + # Reset owner and group to root:root. This mimics what # 'git archive' does and makes the sdist reproducible upon # being built by different users. @@ -910,7 +913,23 @@ def sdist(self, directory: Path) -> pathlib.Path: member = tarfile.TarInfo(f'{dist_name}/PKG-INFO') member.uid = member.gid = 0 member.uname = member.gname = 'root' - member.mtime = time.time() + + # Set the 'PKG-INFO' modification time to the modification time of + # 'pyproject.toml' in the archive generated by 'meson dist'. In + # turn this is the last commit time, unless touched by a dist + # script. This makes the sdist reproducible upon being built at + # different times, when dist scripts are not used, which should be + # the majority of cases. + # + # Note that support for dynamic version in project metadata allows + # the version to depend on the build time. Therefore, setting the + # 'PKG-INFO' modification time to the 'pyproject.toml' + # modification time can be seen as not strictly correct. However, + # the sdist standard does not dictate which modification time to + # use for 'PKG-INFO'. This choice allows to make the sdist + # byte-for-byte reproducible in the most common case. + member.mtime = pyproject_toml_mtime + metadata = bytes(self._metadata.as_rfc822()) member.size = len(metadata) sdist.addfile(member, io.BytesIO(metadata)) diff --git a/mesonpy/_util.py b/mesonpy/_util.py index cdff811c9..c7872a265 100644 --- a/mesonpy/_util.py +++ b/mesonpy/_util.py @@ -37,7 +37,11 @@ def create_targz(path: Path) -> Iterator[tarfile.TarFile]: os.makedirs(os.path.dirname(path), exist_ok=True) file = typing.cast(IO[bytes], gzip.GzipFile( path, - mode='wb', + mode='w', + # Set the stream last modification time to 0. This mimics + # what 'git archive' does and makes the archives byte-for-byte + # reproducible. + mtime=0, )) tar = tarfile.TarFile( mode='w', diff --git a/tests/test_sdist.py b/tests/test_sdist.py index 0be265dd0..e4dddb8c7 100644 --- a/tests/test_sdist.py +++ b/tests/test_sdist.py @@ -9,6 +9,7 @@ import sys import tarfile import textwrap +import time import pytest @@ -205,3 +206,15 @@ def test_long_path(sdist_long_path): 'long_path-1.0.0/meson.build', 'long_path-1.0.0/pyproject.toml' } + + +def test_reproducible(package_pure, tmp_path): + t1 = time.time() + sdist_path_a = mesonpy.build_sdist(tmp_path / 'a') + t2 = time.time() + # Ensure that the two sdists are build at least one second apart. + time.sleep(max(t1 + 1.0 - t2, 0.0)) + sdist_path_b = mesonpy.build_sdist(tmp_path / 'b') + + assert sdist_path_a == sdist_path_b + assert tmp_path.joinpath('a', sdist_path_a).read_bytes() == tmp_path.joinpath('b', sdist_path_b).read_bytes() From 6d590553a4301d6f5b02223ce52156cb1c7035b2 Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Tue, 23 Apr 2024 11:36:20 +0200 Subject: [PATCH 3/4] DOC: remove release process description from published documentation It is not relevant for the users and takes precious space in the documentation index, which is getting quite long. --- .../release-process.rst => RELEASE.rst | 26 ++++++------------- docs/contributing/index.rst | 11 -------- docs/index.rst | 1 - 3 files changed, 8 insertions(+), 30 deletions(-) rename docs/contributing/release-process.rst => RELEASE.rst (68%) delete mode 100644 docs/contributing/index.rst diff --git a/docs/contributing/release-process.rst b/RELEASE.rst similarity index 68% rename from docs/contributing/release-process.rst rename to RELEASE.rst index a530a2402..67963e3b0 100644 --- a/docs/contributing/release-process.rst +++ b/RELEASE.rst @@ -2,20 +2,16 @@ .. .. SPDX-License-Identifier: MIT - -.. _contributing-release-process: - -*************** Release Process -*************** +=============== -All releases are PGP signed with one of the keys listed in the -`installation page`_. Before releasing please make sure your PGP key is listed -there, and preferably signed by one of the other key holders. +All releases are PGP signed with one of the keys listed in ``docs/about.rst``. +Before releasing please make sure your PGP key is listed there, and preferably +signed by one of the other key holders. -If your key is not signed by one of the other key holders, please make sure the -PR that added your key to the :doc:`../about` page was approved by at least one -other maintainer. +If your key is not signed by one of the other key holders, please make sure +that the PR that added your key to ``docs/about.rst`` was approved by at least +one other maintainer. After that is done, you may release the project by following these steps: @@ -44,7 +40,7 @@ After that is done, you may release the project by following these steps: $ git push $ git push --tags -#. Release to `PyPI `_ +#. Release to PyPI: #. Build the Python artifacts: @@ -60,9 +56,3 @@ After that is done, you may release the project by following these steps: There is no need to GPG-sign the artifacts: PyPI no longer supports uploading GPG signatures. - -If you have any questions, please look at previous releases and/or ping the -other maintainers. - - -.. _installation page: installation diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst deleted file mode 100644 index 71ba383e0..000000000 --- a/docs/contributing/index.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. SPDX-FileCopyrightText: 2023 The meson-python developers -.. -.. SPDX-License-Identifier: MIT - - -Development -=========== - -.. toctree:: - - release-process diff --git a/docs/index.rst b/docs/index.rst index 41c7be251..148fea8d3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -100,7 +100,6 @@ the use of ``meson-python`` and Meson for Python packaging. changelog about - contributing/index Discussions Source Code Issue Tracker From 2936b4d44c50004e614760e315b5398b6ba863a7 Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Sun, 3 Mar 2024 10:02:56 +0100 Subject: [PATCH 4/4] DOC: document how to create source distributions --- docs/explanations/design-old.rst | 15 --------- docs/how-to-guides/sdist.rst | 54 ++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 3 files changed, 55 insertions(+), 15 deletions(-) create mode 100644 docs/how-to-guides/sdist.rst diff --git a/docs/explanations/design-old.rst b/docs/explanations/design-old.rst index d9422dd33..dcbb66465 100644 --- a/docs/explanations/design-old.rst +++ b/docs/explanations/design-old.rst @@ -31,21 +31,6 @@ Python tool (pip_, `pypa/build`_, etc.) to build and install the project. ``meson-python`` will build a Python sdist (source distribution) or wheel (binary distribution) from Meson_ project. -Source distribution (sdist) ---------------------------- - -The source distribution is based on ``meson dist``, so make sure all your files -are included there. In git projects, Meson_ will not include files that are not -checked into git, keep that in mind when developing. By default, all files -under version control will be included in the sdist. In order to exclude files, -use ``export-ignore`` or ``export-subst`` attributes in ``.gitattributes`` (see -the ``git-archive`` documentation for details; ``meson dist`` uses -``git-archive`` under the hood). - -Local (uncommitted) changes to files that are under version control will be -included. This is often needed when applying patches, e.g. for build issues -found during packaging, to work around test failures, to amend the license for -vendored components in wheels, etc. Binary distribution (wheels) ---------------------------- diff --git a/docs/how-to-guides/sdist.rst b/docs/how-to-guides/sdist.rst new file mode 100644 index 000000000..76e5731df --- /dev/null +++ b/docs/how-to-guides/sdist.rst @@ -0,0 +1,54 @@ +.. SPDX-FileCopyrightText: 2024 The meson-python developers +.. +.. SPDX-License-Identifier: MIT + +.. _sdist: + +****************************** +Creating a source distribution +****************************** + +A source distribution for the project can be created executing + +.. code-block:: console + + $ python -m build --sdist . + +in the project root folder. This will create a ``.tar.gz`` archive in the +``dist`` folder in the project root folder. This archive contains the full +contents of the latest commit in revision control with all revision control +metadata removed. Uncommitted modifications and files unknown to the revision +control system are not included. + +The source distribution archive is created by adding the required metadata +files to the archive obtained by executing the ``meson dist --no-tests +--allow-dirty`` command. To generate a source distribution, ``meson-python`` +must successfully configure the Meson project by running the ``meson setup`` +command. Additional arguments can be passed to ``meson dist`` to alter its +behavior. Refer to the relevant `Meson documentation`__ and to the +:ref:`how-to-guides-meson-args` guide for details. + +The ``meson dist`` command uses the archival tool of the underlying revision +control system for creating the archive. This implies that a source +distribution can only be created for a project versioned in a revision control +system. Meson supports the Git and Mercurial revision control systems. + +Files can be excluded from the source distribution via the relevant mechanism +provided by the revision control system. When using Git as a revision control +system, it is possible to exclude files from the source distribution setting +the ``export-ignore`` attribute. For example, adding a ``.gitattributes`` +files containing + +.. code-block:: none + + dev/** export-ignore + +would result in the ``dev`` folder to be excluded from the source +distribution. Refer to the ``git archive`` documentation__ for +details. Another mechanism to alter the content of the source distribution is +offered by dist scripts. Refer to the relevant `Meson documentation`__ for +details. + +__ https://mesonbuild.com/Creating-releases.html +__ https://git-scm.com/docs/git-archive#ATTRIBUTES +__ https://mesonbuild.com/Reference-manual_builtin_meson.html#mesonadd_dist_script diff --git a/docs/index.rst b/docs/index.rst index 148fea8d3..1835e19e6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -77,6 +77,7 @@ the use of ``meson-python`` and Meson for Python packaging. :hidden: tutorials/introduction + how-to-guides/sdist how-to-guides/editable-installs how-to-guides/config-settings how-to-guides/meson-args