diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 35d5d59b762660..7dbbe71b2131e7 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -54,9 +54,7 @@ jobs: - x86_64-apple-darwin/clang - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc - - x86_64-unknown-linux-gnu/clang - aarch64-unknown-linux-gnu/gcc - - aarch64-unknown-linux-gnu/clang debug: - true - false @@ -66,41 +64,24 @@ jobs: - target: i686-pc-windows-msvc/msvc architecture: Win32 runner: windows-latest - compiler: msvc - target: x86_64-pc-windows-msvc/msvc architecture: x64 runner: windows-latest - compiler: msvc - target: aarch64-pc-windows-msvc/msvc architecture: ARM64 runner: windows-latest - compiler: msvc - target: x86_64-apple-darwin/clang architecture: x86_64 runner: macos-13 - compiler: clang - target: aarch64-apple-darwin/clang architecture: aarch64 runner: macos-14 - compiler: clang - target: x86_64-unknown-linux-gnu/gcc architecture: x86_64 runner: ubuntu-22.04 - compiler: gcc - - target: x86_64-unknown-linux-gnu/clang - architecture: x86_64 - runner: ubuntu-22.04 - compiler: clang - target: aarch64-unknown-linux-gnu/gcc architecture: aarch64 runner: ubuntu-22.04 - compiler: gcc - - target: aarch64-unknown-linux-gnu/clang - architecture: aarch64 - runner: ubuntu-22.04 - compiler: clang - env: - CC: ${{ matrix.compiler }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -111,10 +92,10 @@ jobs: if: runner.os == 'Windows' && matrix.architecture != 'ARM64' run: | choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} + ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # No PGO or tests (yet): + # No tests (yet): - name: Emulated Windows if: runner.os == 'Windows' && matrix.architecture == 'ARM64' run: | @@ -132,7 +113,7 @@ jobs: find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -141,7 +122,7 @@ jobs: run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -156,13 +137,11 @@ jobs: make clean --jobs 4 export HOST=${{ matrix.architecture }}-linux-gnu sudo apt install --yes "gcc-$HOST" qemu-user - ${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }} - ${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }} export QEMU_LD_PREFIX="/usr/$HOST" - CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \ - CPP="$CC --preprocess" \ + CC="$HOST-gcc" \ + CPP="$HOST-gcc --preprocess" \ HOSTRUNNER=qemu-${{ matrix.architecture }} \ - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index d7257b9d2d00db..5660a6c5105b9f 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -1,5 +1,7 @@ +import json import os import sys +import types from sysconfig import ( _ALWAYS_STR, _PYTHON_BUILD, @@ -157,6 +159,19 @@ def _print_config_dict(d, stream): print ("}", file=stream) +def _get_pybuilddir(): + pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' + if hasattr(sys, "gettotalrefcount"): + pybuilddir += '-pydebug' + return pybuilddir + + +def _get_json_data_name(): + name = _get_sysconfigdata_name() + assert name.startswith('_sysconfigdata') + return name.replace('_sysconfigdata', '_sysconfig_vars') + '.json' + + def _generate_posix_vars(): """Generate the Python module containing build-time variables.""" vars = {} @@ -185,6 +200,8 @@ def _generate_posix_vars(): if _PYTHON_BUILD: vars['BLDSHARED'] = vars['LDSHARED'] + name = _get_sysconfigdata_name() + # There's a chicken-and-egg situation on OS X with regards to the # _sysconfigdata module after the changes introduced by #15298: # get_config_vars() is called by get_platform() as part of the @@ -196,16 +213,13 @@ def _generate_posix_vars(): # _sysconfigdata module manually and populate it with the build vars. # This is more than sufficient for ensuring the subsequent call to # get_platform() succeeds. - name = _get_sysconfigdata_name() - if 'darwin' in sys.platform: - import types - module = types.ModuleType(name) - module.build_time_vars = vars - sys.modules[name] = module + # GH-127178: Since we started generating a .json file, we also need this to + # be able to run sysconfig.get_config_vars(). + module = types.ModuleType(name) + module.build_time_vars = vars + sys.modules[name] = module - pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' - if hasattr(sys, "gettotalrefcount"): - pybuilddir += '-pydebug' + pybuilddir = _get_pybuilddir() os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') @@ -215,6 +229,11 @@ def _generate_posix_vars(): f.write('build_time_vars = ') _print_config_dict(vars, stream=f) + # Write a JSON file with the output of sysconfig.get_config_vars + jsonfile = os.path.join(pybuilddir, _get_json_data_name()) + with open(jsonfile, 'w') as f: + json.dump(get_config_vars(), f, indent=2) + # Create file used for sys.path fixup -- see Modules/getpath.c with open('pybuilddir.txt', 'w', encoding='utf8') as f: f.write(pybuilddir) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index a705dd0cd89e4d..0df1a67ea2b720 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -11,6 +11,7 @@ from test.support import ( captured_stdout, + is_android, is_apple_mobile, is_wasi, PythonSymlink, @@ -25,8 +26,9 @@ from sysconfig import (get_paths, get_platform, get_config_vars, get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, - _expand_vars, _get_preferred_schemes) -from sysconfig.__main__ import _main, _parse_makefile + _expand_vars, _get_preferred_schemes, + is_python_build, _PROJECT_BASE) +from sysconfig.__main__ import _main, _parse_makefile, _get_pybuilddir, _get_json_data_name import _imp import _osx_support import _sysconfig @@ -39,6 +41,7 @@ class TestSysConfig(unittest.TestCase): def setUp(self): super(TestSysConfig, self).setUp() + self.maxDiff = None self.sys_path = sys.path[:] # patching os.uname if hasattr(os, 'uname'): @@ -625,6 +628,32 @@ def test_makefile_overwrites_config_vars(self): self.assertNotEqual(data['prefix'], data['base_prefix']) self.assertNotEqual(data['exec_prefix'], data['base_exec_prefix']) + @unittest.skipIf(os.name != 'posix', '_sysconfig-vars JSON file is only available on POSIX') + @unittest.skipIf(is_wasi, "_sysconfig-vars JSON file currently isn't available on WASI") + @unittest.skipIf(is_android or is_apple_mobile, 'Android and iOS change the prefix') + def test_sysconfigdata_json(self): + if '_PYTHON_SYSCONFIGDATA_PATH' in os.environ: + data_dir = os.environ['_PYTHON_SYSCONFIGDATA_PATH'] + elif is_python_build(): + data_dir = os.path.join(_PROJECT_BASE, _get_pybuilddir()) + else: + data_dir = sys._stdlib_dir + + json_data_path = os.path.join(data_dir, _get_json_data_name()) + + with open(json_data_path) as f: + json_config_vars = json.load(f) + + system_config_vars = get_config_vars() + + # Ignore keys in the check + for key in ('projectbase', 'srcdir'): + json_config_vars.pop(key) + system_config_vars.pop(key) + + self.assertEqual(system_config_vars, json_config_vars) + + class MakefileTests(unittest.TestCase): @unittest.skipIf(sys.platform.startswith('win'), diff --git a/Makefile.pre.in b/Makefile.pre.in index 8d94ba361fd934..724354746b8d81 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2645,8 +2645,8 @@ libinstall: all $(srcdir)/Modules/xxmodule.c esac; \ done; \ done - $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py \ - $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfig_vars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt @ # If app store compliance has been configured, apply the patch to the @ # installed library code. The patch has been previously validated against diff --git a/Misc/NEWS.d/next/Library/2024-11-26-17-42-00.gh-issue-127178.U8hxjc.rst b/Misc/NEWS.d/next/Library/2024-11-26-17-42-00.gh-issue-127178.U8hxjc.rst new file mode 100644 index 00000000000000..b703b58ea8e1d9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-26-17-42-00.gh-issue-127178.U8hxjc.rst @@ -0,0 +1,4 @@ +A ``_sysconfig_vars_(...).json`` file is now shipped in the standard library +directory. It contains the output of :func:`sysconfig.get_config_vars` on +the default environment encoded as JSON data. This is an implementation +detail, and may change at any time.