windows: support building CPython 3.8

The remaining work to build 3.8 was mostly related to building
libffi. This is definitely one of the wonkier dependencies
to build from source, especially since the source release
published by Python lacks a commit necessary to make the
build not error. I wasted a few hours tracking this down. *sigh*.

After a few false starts, including reimplementing aspects of the
`prepare_libffi.bat` script, I gave up and just used it verbatim.
We may come to regret this shortcut to support static builds,
which don't yet work for non-ffi reasons. But at least
`--profile shared` works!

libffi requiring cygwin is a bit annoying. But what are you going
to do?

Once libffi was being built, we encountered a packaging error
due to a new test extension module. So that change snuck in as
well.

Author: indygreg

cpython-windows/build.py

@@ -150,7 +150,7 @@ def find_vswhere():
     return vswhere
 
 
-def find_msbuild():
+def find_vs_path(path):
     vswhere = find_vswhere()
 
     p = subprocess.check_output(
@@ -169,7 +169,7 @@ def find_msbuild():
     # Strictly speaking the output may not be UTF-8.
     p = pathlib.Path(p.strip().decode("utf-8"))
 
-    p = p / "MSBuild" / "Current" / "Bin" / "MSBuild.exe"
+    p = p / path
 
     if not p.exists():
         print("%s does not exist" % p)
@@ -178,6 +178,15 @@ def find_msbuild():
     return p
 
 
+def find_msbuild():
+    return find_vs_path(pathlib.Path("MSBuild") / "Current" / "Bin" / "MSBuild.exe")
+
+
+def find_vcvarsall_path():
+    """Find path to vcvarsall.bat"""
+    return find_vs_path(pathlib.Path("VC") / "Auxiliary" / "Build" / "vcvarsall.bat")
+
+
 def find_vctools_path():
     vswhere = find_vswhere()
 
@@ -1071,6 +1080,9 @@ def run_msbuild(
         "/property:OverrideVersion=%s" % python_version,
     ]
 
+    if not python_version.startswith("3.7"):
+        args.append("/property:IncludeCTypes=true")
+
     exec_and_log(args, str(pcbuild_path), os.environ)
 
 
@@ -1194,6 +1206,50 @@ def build_openssl(perl_path: pathlib.Path, arch: str, profile: str):
             create_tar_from_directory(fh, install)
 
 
+def build_libffi(
+    build_dir: pathlib.Path, arch: str, prepare_ffi: pathlib.Path, sh_exe: pathlib.Path
+):
+    ffi_source_path = build_dir / "libffi"
+
+    # As of April 15, 2020, the libffi source release on GitHub doesn't
+    # have patches that we need to build. https://bugs.python.org/issue40293
+    # tracks getting a proper release. Until then, git clone the repo.
+    subprocess.run(
+        [
+            "git.exe",
+            "clone",
+            "--single-branch",
+            "--branch",
+            "libffi",
+            "https://github.com/python/cpython-source-deps.git",
+            str(ffi_source_path),
+        ],
+        check=True,
+    )
+
+    subprocess.run(
+        ["git.exe", "checkout", "ed22026f39b37f892ded95d7b30e77dfb5126334"],
+        cwd=ffi_source_path,
+        check=True,
+    )
+
+    # We build libffi by running the build script that CPython ships.
+    env = dict(os.environ)
+    env["LIBFFI_SOURCE"] = str(ffi_source_path)
+    env["VCVARSALL"] = str(find_vcvarsall_path())
+    env["SH"] = str(sh_exe)
+
+    args = [str(prepare_ffi), "-pdb"]
+    if arch == "x86":
+        args.append("-x86")
+    else:
+        args.append("-x64")
+
+    # Running the build script from Python will install the files into the
+    # appropriate directory.
+    subprocess.run(args, env=env, check=True)
+
+
 RE_ADDITIONAL_DEPENDENCIES = re.compile(
     "<AdditionalDependencies>([^<]+)</AdditionalDependencies>"
 )
@@ -1249,6 +1305,7 @@ def collect_python_build_artifacts(
         "_testconsole",
         "_testembed",
         "_testimportmultiple",
+        "_testinternalcapi",
         "_testmultiphase",
         "xxlimited",
     }
@@ -1481,7 +1538,7 @@ def find_additional_dependencies(project: pathlib.Path):
     return res
 
 
-def build_cpython(python_entry_name: str, arch: str, profile):
+def build_cpython(python_entry_name: str, arch: str, sh_exe, profile):
     static = profile == "static"
     pgo = "-pgo" in profile
 
@@ -1545,6 +1602,16 @@ def build_cpython(python_entry_name: str, arch: str, profile):
         with zipfile.ZipFile(setuptools_archive) as zf:
             zf.extractall(td)
 
+        cpython_source_path = td / ("Python-%s" % python_version)
+        pcbuild_path = cpython_source_path / "PCBuild"
+
+        prepare_libffi = pcbuild_path / "prepare_libffi.bat"
+
+        if prepare_libffi.exists():
+            assert sh_exe
+
+            build_libffi(td, arch, prepare_libffi, sh_exe)
+
         # We need all the OpenSSL library files in the same directory to appease
         # install rules.
         if not static:
@@ -1879,9 +1946,14 @@ def main():
         default="static",
         help="How to compile Python",
     )
+    parser.add_argument("--sh", help="Path to sh.exe in a cygwin or mingw installation")
 
     args = parser.parse_args()
 
+    if args.python == "cpython-3.8" and not args.sh:
+        print("--sh required when building Python 3.8+")
+        return 1
+
     now = datetime.datetime.utcnow()
 
     log_path = BUILD / "build.log"
@@ -1899,7 +1971,9 @@ def main():
             build_openssl(perl_path, arch, profile=args.profile)
 
         LOG_PREFIX[0] = "cpython"
-        tar_path = build_cpython(args.python, arch, profile=args.profile)
+        tar_path = build_cpython(
+            args.python, arch, sh_exe=pathlib.Path(args.sh), profile=args.profile
+        )
 
         compress_python_archive(
             tar_path, DIST, "%s-%s" % (tar_path.stem, now.strftime("%Y%m%dT%H%M")),

docs/building.rst

@@ -52,16 +52,27 @@ Windows
 Visual Studio 2017 (or later) is required. A compatible Windows SDK is required
 (10.0.17763.0 as per CPython 3.7.2).
 
+If building CPython 3.8+, there are the following additional requirements:
+
+* A ``git.exe`` on ``PATH`` (to clone ``libffi`` from source).
+* An installation of Cywgin with the ``autoconf``, ``automake``, ``libtool``,
+  and ``make`` packages installed. (``libffi`` build dependency.)
+
 To build a Python distribution for Windows x64::
 
-   # From a Visual Studio 2017 x64 native tools command prompt:
-   $ py.exe build-windows.py
+   # From a Visual Studio 2017/2019 x64 native tools command prompt:
+   $ py.exe build-windows.py --profile static
 
 It is also possible to build a more traditional dynamically linked
 distribution, optionally with PGO optimizations::
 
    $ py.exe build-windows.py --profile shared
    $ py.exe build-windows.py --profile shared-pgo
 
+If building CPython 3.8+, you will need to specify the path to a
+``sh.exe`` installed from cygwin. e.g.
+
+   $ py.exe build-windows.py --python cpython-3.8 --sh c:\cygwin\bin\sh.exe --profile shared
+
 To build a 32-bit x86 binary, simply use an ``x86 Native Tools
 Command Prompt`` instead of ``x64``.