diff --git a/docs/contributing/release-process.rst b/RELEASE.rst similarity index 68% rename from docs/contributing/release-process.rst rename to RELEASE.rst index a530a2402..67963e3b0 100644 --- a/docs/contributing/release-process.rst +++ b/RELEASE.rst @@ -2,20 +2,16 @@ .. .. SPDX-License-Identifier: MIT - -.. _contributing-release-process: - -*************** Release Process -*************** +=============== -All releases are PGP signed with one of the keys listed in the -`installation page`_. Before releasing please make sure your PGP key is listed -there, and preferably signed by one of the other key holders. +All releases are PGP signed with one of the keys listed in ``docs/about.rst``. +Before releasing please make sure your PGP key is listed there, and preferably +signed by one of the other key holders. -If your key is not signed by one of the other key holders, please make sure the -PR that added your key to the :doc:`../about` page was approved by at least one -other maintainer. +If your key is not signed by one of the other key holders, please make sure +that the PR that added your key to ``docs/about.rst`` was approved by at least +one other maintainer. After that is done, you may release the project by following these steps: @@ -44,7 +40,7 @@ After that is done, you may release the project by following these steps: $ git push $ git push --tags -#. Release to `PyPI `_ +#. Release to PyPI: #. Build the Python artifacts: @@ -60,9 +56,3 @@ After that is done, you may release the project by following these steps: There is no need to GPG-sign the artifacts: PyPI no longer supports uploading GPG signatures. - -If you have any questions, please look at previous releases and/or ping the -other maintainers. - - -.. _installation page: installation diff --git a/docs/contributing/index.rst b/docs/contributing/index.rst deleted file mode 100644 index 71ba383e0..000000000 --- a/docs/contributing/index.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. SPDX-FileCopyrightText: 2023 The meson-python developers -.. -.. SPDX-License-Identifier: MIT - - -Development -=========== - -.. toctree:: - - release-process diff --git a/docs/explanations/design-old.rst b/docs/explanations/design-old.rst index d9422dd33..dcbb66465 100644 --- a/docs/explanations/design-old.rst +++ b/docs/explanations/design-old.rst @@ -31,21 +31,6 @@ Python tool (pip_, `pypa/build`_, etc.) to build and install the project. ``meson-python`` will build a Python sdist (source distribution) or wheel (binary distribution) from Meson_ project. -Source distribution (sdist) ---------------------------- - -The source distribution is based on ``meson dist``, so make sure all your files -are included there. In git projects, Meson_ will not include files that are not -checked into git, keep that in mind when developing. By default, all files -under version control will be included in the sdist. In order to exclude files, -use ``export-ignore`` or ``export-subst`` attributes in ``.gitattributes`` (see -the ``git-archive`` documentation for details; ``meson dist`` uses -``git-archive`` under the hood). - -Local (uncommitted) changes to files that are under version control will be -included. This is often needed when applying patches, e.g. for build issues -found during packaging, to work around test failures, to amend the license for -vendored components in wheels, etc. Binary distribution (wheels) ---------------------------- diff --git a/docs/how-to-guides/sdist.rst b/docs/how-to-guides/sdist.rst new file mode 100644 index 000000000..76e5731df --- /dev/null +++ b/docs/how-to-guides/sdist.rst @@ -0,0 +1,54 @@ +.. SPDX-FileCopyrightText: 2024 The meson-python developers +.. +.. SPDX-License-Identifier: MIT + +.. _sdist: + +****************************** +Creating a source distribution +****************************** + +A source distribution for the project can be created executing + +.. code-block:: console + + $ python -m build --sdist . + +in the project root folder. This will create a ``.tar.gz`` archive in the +``dist`` folder in the project root folder. This archive contains the full +contents of the latest commit in revision control with all revision control +metadata removed. Uncommitted modifications and files unknown to the revision +control system are not included. + +The source distribution archive is created by adding the required metadata +files to the archive obtained by executing the ``meson dist --no-tests +--allow-dirty`` command. To generate a source distribution, ``meson-python`` +must successfully configure the Meson project by running the ``meson setup`` +command. Additional arguments can be passed to ``meson dist`` to alter its +behavior. Refer to the relevant `Meson documentation`__ and to the +:ref:`how-to-guides-meson-args` guide for details. + +The ``meson dist`` command uses the archival tool of the underlying revision +control system for creating the archive. This implies that a source +distribution can only be created for a project versioned in a revision control +system. Meson supports the Git and Mercurial revision control systems. + +Files can be excluded from the source distribution via the relevant mechanism +provided by the revision control system. When using Git as a revision control +system, it is possible to exclude files from the source distribution setting +the ``export-ignore`` attribute. For example, adding a ``.gitattributes`` +files containing + +.. code-block:: none + + dev/** export-ignore + +would result in the ``dev`` folder to be excluded from the source +distribution. Refer to the ``git archive`` documentation__ for +details. Another mechanism to alter the content of the source distribution is +offered by dist scripts. Refer to the relevant `Meson documentation`__ for +details. + +__ https://mesonbuild.com/Creating-releases.html +__ https://git-scm.com/docs/git-archive#ATTRIBUTES +__ https://mesonbuild.com/Reference-manual_builtin_meson.html#mesonadd_dist_script diff --git a/docs/index.rst b/docs/index.rst index 41c7be251..1835e19e6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -77,6 +77,7 @@ the use of ``meson-python`` and Meson for Python packaging. :hidden: tutorials/introduction + how-to-guides/sdist how-to-guides/editable-installs how-to-guides/config-settings how-to-guides/meson-args @@ -100,7 +101,6 @@ the use of ``meson-python`` and Meson for Python packaging. changelog about - contributing/index Discussions Source Code Issue Tracker diff --git a/mesonpy/__init__.py b/mesonpy/__init__.py index 0455bfd19..83653c2ce 100644 --- a/mesonpy/__init__.py +++ b/mesonpy/__init__.py @@ -31,7 +31,6 @@ import tarfile import tempfile import textwrap -import time import typing import warnings @@ -865,64 +864,77 @@ def _meson_version(self) -> str: def sdist(self, directory: Path) -> pathlib.Path: """Generates a sdist (source distribution) in the specified directory.""" - # generate meson dist file + # Generate meson dist file. self._run(self._meson + ['dist', '--allow-dirty', '--no-tests', '--formats', 'gztar', *self._meson_args['dist']]) - # move meson dist file to output path dist_name = f'{self._metadata.distribution_name}-{self._metadata.version}' meson_dist_name = f'{self._meson_name}-{self._meson_version}' meson_dist_path = pathlib.Path(self._build_dir, 'meson-dist', f'{meson_dist_name}.tar.gz') - sdist = pathlib.Path(directory, f'{dist_name}.tar.gz') + sdist_path = pathlib.Path(directory, f'{dist_name}.tar.gz') + pyproject_toml_mtime = 0 - with tarfile.open(meson_dist_path, 'r:gz') as meson_dist, mesonpy._util.create_targz(sdist) as tar: + with tarfile.open(meson_dist_path, 'r:gz') as meson_dist, mesonpy._util.create_targz(sdist_path) as sdist: for member in meson_dist.getmembers(): - # calculate the file path in the source directory - assert member.name, member.name - member_parts = member.name.split('/') - if len(member_parts) <= 1: - continue - path = self._source_dir.joinpath(*member_parts[1:]) - - if not path.exists() and member.isfile(): - # File doesn't exists on the source directory but exists on - # the Meson dist, so it is generated file, which we need to - # include. - # See https://mesonbuild.com/Reference-manual_builtin_meson.html#mesonadd_dist_script - - # MESON_DIST_ROOT could have a different base name - # than the actual sdist basename, so we need to rename here + if member.isfile(): file = meson_dist.extractfile(member.name) - member.name = str(pathlib.Path(dist_name, *member_parts[1:]).as_posix()) - tar.addfile(member, file) - continue - if not path.is_file(): - continue + # Reset pax extended header. The tar archive member may be + # using pax headers to store some file metadata. The pax + # headers are not reset when the metadata is modified and + # they take precedence when the member is deserialized. + # This is relevant because when rewriting the member name, + # the length of the path may shrink from being more than + # 100 characters (requiring the path to be stored in the + # pax headers) to being less than 100 characters. When this + # happens, the tar archive member is serialized with the + # shorter name in the regular header and the longer one in + # the extended pax header. The archives handled here are + # not expected to use extended pax headers other than for + # the ones required to encode file metadata. The easiest + # solution is to reset the pax extended headers. + member.pax_headers = {} + + # Rewrite the path to match the sdist distribution name. + stem = member.name.split('/', 1)[1] + member.name = '/'.join((dist_name, stem)) + + if stem == 'pyproject.toml': + pyproject_toml_mtime = member.mtime + + # Reset owner and group to root:root. This mimics what + # 'git archive' does and makes the sdist reproducible upon + # being built by different users. + member.uname = member.gname = 'root' + member.uid = member.gid = 0 + + sdist.addfile(member, file) + + # Add 'PKG-INFO'. + member = tarfile.TarInfo(f'{dist_name}/PKG-INFO') + member.uid = member.gid = 0 + member.uname = member.gname = 'root' + + # Set the 'PKG-INFO' modification time to the modification time of + # 'pyproject.toml' in the archive generated by 'meson dist'. In + # turn this is the last commit time, unless touched by a dist + # script. This makes the sdist reproducible upon being built at + # different times, when dist scripts are not used, which should be + # the majority of cases. + # + # Note that support for dynamic version in project metadata allows + # the version to depend on the build time. Therefore, setting the + # 'PKG-INFO' modification time to the 'pyproject.toml' + # modification time can be seen as not strictly correct. However, + # the sdist standard does not dictate which modification time to + # use for 'PKG-INFO'. This choice allows to make the sdist + # byte-for-byte reproducible in the most common case. + member.mtime = pyproject_toml_mtime - info = tarfile.TarInfo(member.name) - file_stat = os.stat(path) - info.mtime = member.mtime - info.size = file_stat.st_size - info.mode = int(oct(file_stat.st_mode)[-3:], 8) - - # rewrite the path if necessary, to match the sdist distribution name - if dist_name != meson_dist_name: - info.name = pathlib.Path( - dist_name, - path.relative_to(self._source_dir) - ).as_posix() - - with path.open('rb') as f: - tar.addfile(info, fileobj=f) - - # add PKG-INFO to dist file to make it a sdist - pkginfo_info = tarfile.TarInfo(f'{dist_name}/PKG-INFO') - pkginfo_info.mtime = time.time() # type: ignore[assignment] metadata = bytes(self._metadata.as_rfc822()) - pkginfo_info.size = len(metadata) - tar.addfile(pkginfo_info, fileobj=io.BytesIO(metadata)) + member.size = len(metadata) + sdist.addfile(member, io.BytesIO(metadata)) - return sdist + return sdist_path def wheel(self, directory: Path) -> pathlib.Path: """Generates a wheel in the specified directory.""" diff --git a/mesonpy/_util.py b/mesonpy/_util.py index cdff811c9..c7872a265 100644 --- a/mesonpy/_util.py +++ b/mesonpy/_util.py @@ -37,7 +37,11 @@ def create_targz(path: Path) -> Iterator[tarfile.TarFile]: os.makedirs(os.path.dirname(path), exist_ok=True) file = typing.cast(IO[bytes], gzip.GzipFile( path, - mode='wb', + mode='w', + # Set the stream last modification time to 0. This mimics + # what 'git archive' does and makes the archives byte-for-byte + # reproducible. + mtime=0, )) tar = tarfile.TarFile( mode='w', diff --git a/tests/packages/long-path/meson.build b/tests/packages/long-path/meson.build new file mode 100644 index 000000000..272d7fe0f --- /dev/null +++ b/tests/packages/long-path/meson.build @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2024 The meson-python developers +# +# SPDX-License-Identifier: MIT + +project('very-long-project-name-that-makes-the-paths-within-the-sdist-exceed-100-characters-xxxxxxxxxxxxxxxxx', version: '1.0.0') diff --git a/tests/packages/long-path/pyproject.toml b/tests/packages/long-path/pyproject.toml new file mode 100644 index 000000000..70791dd42 --- /dev/null +++ b/tests/packages/long-path/pyproject.toml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2021 The meson-python developers +# +# SPDX-License-Identifier: MIT + +[build-system] +build-backend = 'mesonpy' +requires = ['meson-python'] + +[project] +name = 'long-path' +dynamic = ['version'] diff --git a/tests/test_sdist.py b/tests/test_sdist.py index fb698b53d..e4dddb8c7 100644 --- a/tests/test_sdist.py +++ b/tests/test_sdist.py @@ -3,11 +3,13 @@ # SPDX-License-Identifier: MIT import os +import pathlib import re import stat import sys import tarfile import textwrap +import time import pytest @@ -122,37 +124,35 @@ def test_contents_subdirs(sdist_subdirs): def test_contents_unstaged(package_pure, tmp_path): - new_data = textwrap.dedent(''' - def bar(): - return 'foo' + new = textwrap.dedent(''' + def bar(): + return 'foo' ''').strip() - with open('pure.py', 'r') as f: - old_data = f.read() - - try: - with in_git_repo_context(): - with open('pure.py', 'w') as f, open('crap', 'x'): - f.write(new_data) + old = pathlib.Path('pure.py').read_text() + with in_git_repo_context(): + try: + pathlib.Path('pure.py').write_text(new) + pathlib.Path('other.py').touch() sdist_path = mesonpy.build_sdist(os.fspath(tmp_path)) - finally: - with open('pure.py', 'w') as f: - f.write(old_data) - os.unlink('crap') + finally: + pathlib.Path('pure.py').write_text(old) + pathlib.Path('other.py').unlink() with tarfile.open(tmp_path / sdist_path, 'r:gz') as sdist: names = {member.name for member in sdist.getmembers()} mtimes = {member.mtime for member in sdist.getmembers()} - read_data = sdist.extractfile('pure-1.0.0/pure.py').read().replace(b'\r\n', b'\n') + data = sdist.extractfile('pure-1.0.0/pure.py').read().replace(b'\r\n', b'\n') + # Verify that uncommitted changes are not included in the sdist. assert names == { 'pure-1.0.0/PKG-INFO', 'pure-1.0.0/meson.build', 'pure-1.0.0/pure.py', 'pure-1.0.0/pyproject.toml', } - assert read_data == new_data.encode() + assert data == old.encode() # All the archive members have a valid mtime. assert 0 not in mtimes @@ -192,3 +192,29 @@ def test_generated_files(sdist_generated_files): # All the archive members have a valid mtime. assert 0 not in mtimes + + +def test_long_path(sdist_long_path): + # See https://github.com/mesonbuild/meson-python/pull/587#pullrequestreview-2020891328 + # and https://github.com/mesonbuild/meson-python/pull/587#issuecomment-2075973593 + + with tarfile.open(sdist_long_path, 'r:gz') as sdist: + names = {member.name for member in sdist.getmembers()} + + assert names == { + 'long_path-1.0.0/PKG-INFO', + 'long_path-1.0.0/meson.build', + 'long_path-1.0.0/pyproject.toml' + } + + +def test_reproducible(package_pure, tmp_path): + t1 = time.time() + sdist_path_a = mesonpy.build_sdist(tmp_path / 'a') + t2 = time.time() + # Ensure that the two sdists are build at least one second apart. + time.sleep(max(t1 + 1.0 - t2, 0.0)) + sdist_path_b = mesonpy.build_sdist(tmp_path / 'b') + + assert sdist_path_a == sdist_path_b + assert tmp_path.joinpath('a', sdist_path_a).read_bytes() == tmp_path.joinpath('b', sdist_path_b).read_bytes()