linux: support building debug builds of Python

Debug builds can be useful for debugging built binaries.

Author: indygreg

build-linux.py

@@ -24,6 +24,7 @@
 
 def bootstrap():
     parser = argparse.ArgumentParser()
+    parser.add_argument('--debug', action='store_true')
     parser.add_argument('--musl', action='store_true')
     parser.add_argument('--optimized', action='store_true')
 
@@ -41,6 +42,8 @@ def bootstrap():
     os.environ['PATH'] = '%s:%s' % (str(VENV / 'bin'), os.environ['PATH'])
     os.environ['PYTHONPATH'] = str(ROOT)
 
+    if args.debug:
+        os.environ['PYBUILD_DEBUG'] = '1'
     if args.musl:
         os.environ['PYBUILD_MUSL'] = '1'
     if args.optimized:
@@ -64,6 +67,9 @@ def run():
     if 'PYBUILD_MUSL' in os.environ:
         basename += '-musl'
         extra = '-musl'
+    if 'PYBUILD_DEBUG' in os.environ:
+        basename += '-debug'
+        extra += '-debug'
     if 'PYBUILD_OPTIMIZED' in os.environ:
         basename += '-pgo'
         extra = '-pgo'

cpython-linux/Makefile

@@ -15,6 +15,16 @@ ifdef PYBUILD_MUSL
     PLATFORM := $(PLATFORM)-musl
 endif
 
+ifdef PYBUILD_DEBUG
+	DEFAULT_EXTRA := -debug
+else
+ifdef PYBUILD_OPTIMIZED
+	DEFAULT_EXTRA := -pgo
+else
+	DEFAULT_EXTRA :=
+endif
+endif
+
 BASE_TOOLCHAIN_DEPENDS := \
     $(OUTDIR)/binutils-linux64.tar \
     $(OUTDIR)/gcc-linux64.tar \
@@ -31,7 +41,7 @@ PYTHON_DEP_DEPENDS := \
     $(TOOLCHAIN_DEPENDS) \
     $(NULL)
 
-default: $(OUTDIR)/cpython-$(PLATFORM)$(if $(PYBUILD_OPTIMIZED),-pgo,).tar
+default: $(OUTDIR)/cpython-$(PLATFORM)$(DEFAULT_EXTRA).tar
 
 $(OUTDIR)/image-%.tar: $(HERE)/%.Dockerfile $(COMMON_DEPENDS)
 	$(BUILD) image-$*
@@ -106,5 +116,8 @@ PYTHON_DEPENDS := \
 $(OUTDIR)/cpython-$(PLATFORM).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-cpython.sh $(PYTHON_DEPENDS)
 	$(BUILD) --platform $(PLATFORM) cpython
 
+$(OUTDIR)/cpython-$(PLATFORM)-debug.tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-cpython.sh $(PYTHON_DEPENDS)
+	$(BUILD) --platform $(PLATFORM) --debug cpython
+
 $(OUTDIR)/cpython-$(PLATFORM)-pgo.tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-cpython.sh $(PYTHON_DEPENDS)
 	$(BUILD) --platform $(PLATFORM) --optimized cpython

cpython-linux/build-cpython.sh

@@ -75,6 +75,10 @@ fi
 
 CONFIGURE_FLAGS="--prefix=/install --with-openssl=/tools/deps --without-ensurepip"
 
+if [ -n "${CPYTHON_DEBUG}" ]; then
+    CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --with-pydebug"
+fi
+
 # TODO support --with-lto
 # --with-lto will produce .o files that are LLVM bitcode and aren't compatible
 # with downstream consumers that can't handle them.

cpython-linux/build.py

@@ -629,7 +629,7 @@ def process_setup_line(line, variant=None):
     return bi
 
 
-def build_cpython(client, image, platform, optimized=False, musl=False):
+def build_cpython(client, image, platform, debug=False, optimized=False, musl=False):
     """Build CPython in a Docker image'"""
     python_archive = download_entry('cpython-3.7', BUILD)
     setuptools_archive = download_entry('setuptools', BUILD)
@@ -639,7 +639,7 @@ def build_cpython(client, image, platform, optimized=False, musl=False):
         static_modules_lines = [l.rstrip() for l in fh if not l.startswith(b'#')]
 
     setup = derive_setup_local(static_modules_lines, python_archive,
-                               musl=musl)
+                               musl=musl, debug=debug)
 
     config_c_in = parse_config_c(setup['config_c_in'].decode('utf-8'))
     setup_dist_content = setup['setup_dist']
@@ -706,21 +706,27 @@ def build_cpython(client, image, platform, optimized=False, musl=False):
         if musl:
             env['CC'] = 'musl-clang'
 
+        if debug:
+            env['CPYTHON_DEBUG'] = '1'
         if optimized:
             env['CPYTHON_OPTIMIZED'] = '1'
 
         container_exec(container, '/build/build-cpython.sh',
                        environment=env)
 
+        fully_qualified_name = 'python%s%sm' % (
+            '3.7', 'd' if debug else ''
+        )
+
         # Create PYTHON.json file describing this distribution.
         python_info = {
             'version': '2',
             'os': 'linux',
             'arch': 'x86_64',
             'python_flavor': 'cpython',
             'python_version': DOWNLOADS['cpython-3.7']['version'],
-            'python_exe': 'install/bin/python3.7',
-            'python_include': 'install/include/python3.7m',
+            'python_exe': 'install/bin/%s' % fully_qualified_name,
+            'python_include': 'install/include/%s' % fully_qualified_name,
             'python_stdlib': 'install/lib/python3.7',
             'build_info': python_build_info(container, config_c_in,
                                             setup_dist_content, setup_local_content),
@@ -740,6 +746,8 @@ def build_cpython(client, image, platform, optimized=False, musl=False):
 
         if musl:
             basename += '-musl'
+        if debug:
+            basename += '-debug'
         if optimized:
             basename += '-pgo'
 
@@ -764,6 +772,7 @@ def main():
         return 1
 
     parser = argparse.ArgumentParser()
+    parser.add_argument('--debug', action='store_true')
     parser.add_argument('--platform')
     parser.add_argument('--optimized', action='store_true')
     parser.add_argument('action')
@@ -775,6 +784,8 @@ def main():
     name = action
     if args.platform:
         name += '-%s' % args.platform
+    if args.debug:
+        name += '-debug'
     if args.optimized:
         name += '-pgo'
 
@@ -823,7 +834,7 @@ def main():
 
         elif action == 'cpython':
             build_cpython(client, get_image(client, 'build'), platform=platform,
-                          musl=musl, optimized=args.optimized)
+                          musl=musl, debug=args.debug, optimized=args.optimized)
 
         else:
             print('unknown build action: %s' % action)

pythonbuild/cpython.py

@@ -116,7 +116,7 @@ def parse_setup_line(line: bytes, variant: str):
 
 
 def derive_setup_local(static_modules_lines, cpython_source_archive, disabled=None,
-                       musl=False):
+                       musl=False, debug=False):
     """Derive the content of the Modules/Setup.local file."""
     python_version = DOWNLOADS['cpython-3.7']['version']
 
@@ -134,6 +134,10 @@ def derive_setup_local(static_modules_lines, cpython_source_archive, disabled=No
         disabled.add(b'nis')
         disabled.add(b'ossaudiodev')
 
+    if debug:
+        # Doesn't work in debug builds.
+        disabled.add(b'xxlimited')
+
     with tarfile.open(str(cpython_source_archive)) as tf:
         ifh = tf.extractfile('Python-%s/Modules/Setup.dist' % python_version)
         source_lines = ifh.readlines()