Rebuild MacOS python with lzma module installed (#975)

Also update `scripts/update_python.py` such that it works on
linux once again.

Fixes: emscripten-core/emscripten#16073

Author: sbc100

emsdk_manifest.json

@@ -422,7 +422,7 @@
   },
   {
     "id": "python",
-    "version": "3.9.2-1",
+    "version": "3.9.2",
     "bitness": 64,
     "arch": "x86_64",
     "windows_url": "python-3.9.2-1-embed-amd64+pywin32.zip",
@@ -431,16 +431,16 @@
   },
   {
     "id": "python",
-    "version": "3.9.2-1",
+    "version": "3.9.2",
     "bitness": 64,
     "arch": "x86_64",
-    "macos_url": "python-3.9.2-1-macos-x86_64.tar.gz",
+    "macos_url": "python-3.9.2-2-macos-x86_64.tar.gz",
     "activated_cfg": "PYTHON='%installation_dir%/bin/python3'",
     "activated_env": "EMSDK_PYTHON=%installation_dir%/bin/python3;SSL_CERT_FILE=%installation_dir%/lib/python3.9/site-packages/certifi/cacert.pem"
   },
   {
     "id": "python",
-    "version": "3.9.2-1",
+    "version": "3.9.2",
     "bitness": 64,
     "arch": "aarch64",
     "macos_url": "python-3.9.2-1-macos-arm64.tar.gz",
@@ -644,13 +644,13 @@
   {
     "version": "upstream-main",
     "bitness": 64,
-    "uses": ["python-3.9.2-1-64bit", "llvm-git-main-64bit", "node-14.18.2-64bit", "emscripten-main-64bit", "binaryen-main-64bit"],
+    "uses": ["python-3.9.2-64bit", "llvm-git-main-64bit", "node-14.18.2-64bit", "emscripten-main-64bit", "binaryen-main-64bit"],
     "os": "win"
   },
   {
     "version": "upstream-main",
     "bitness": 64,
-    "uses": ["python-3.9.2-1-64bit", "llvm-git-main-64bit", "node-14.18.2-64bit", "emscripten-main-64bit", "binaryen-main-64bit"],
+    "uses": ["python-3.9.2-64bit", "llvm-git-main-64bit", "node-14.18.2-64bit", "emscripten-main-64bit", "binaryen-main-64bit"],
     "os": "macos"
   },
   {
@@ -711,23 +711,23 @@
   {
     "version": "releases-upstream-%releases-tag%",
     "bitness": 64,
-    "uses": ["node-14.18.2-64bit", "python-3.9.2-1-64bit", "releases-upstream-%releases-tag%-64bit"],
+    "uses": ["node-14.18.2-64bit", "python-3.9.2-64bit", "releases-upstream-%releases-tag%-64bit"],
     "os": "macos",
     "arch": "x86_64",
     "custom_install_script": "emscripten_npm_install"
   },
   {
     "version": "releases-upstream-%releases-tag%",
     "bitness": 64,
-    "uses": ["node-14.18.2-64bit", "python-3.9.2-1-64bit", "releases-upstream-%releases-tag%-64bit"],
+    "uses": ["node-14.18.2-64bit", "python-3.9.2-64bit", "releases-upstream-%releases-tag%-64bit"],
     "os": "macos",
     "arch": "aarch64",
     "custom_install_script": "emscripten_npm_install"
   },
   {
     "version": "releases-upstream-%releases-tag%",
     "bitness": 64,
-    "uses": ["node-14.18.2-64bit", "python-3.9.2-1-64bit", "java-8.152-64bit", "releases-upstream-%releases-tag%-64bit"],
+    "uses": ["node-14.18.2-64bit", "python-3.9.2-64bit", "java-8.152-64bit", "releases-upstream-%releases-tag%-64bit"],
     "os": "win",
     "custom_install_script": "emscripten_npm_install"
   },
@@ -749,7 +749,7 @@
   {
     "version": "releases-fastcomp-%releases-tag%",
     "bitness": 64,
-    "uses": ["node-14.18.2-64bit", "python-3.9.2-1-64bit", "releases-fastcomp-%releases-tag%-64bit"],
+    "uses": ["node-14.18.2-64bit", "python-3.9.2-64bit", "releases-fastcomp-%releases-tag%-64bit"],
     "os": "macos",
     "arch": "aarch64",
     "custom_install_script": "emscripten_npm_install"

scripts/update_python.py

@@ -34,7 +34,7 @@
 version = '3.9.2'
 major_minor_version = '.'.join(version.split('.')[:2])  # e.g. '3.9.2' -> '3.9'
 base = 'https://www.python.org/ftp/python/%s/' % version
-revision = '1'
+revision = '2'
 
 pywin32_version = '227'
 pywin32_base = 'https://github.com/mhammond/pywin32/releases/download/b%s/' % pywin32_version
@@ -105,19 +105,22 @@ def build_python():
     if sys.platform.startswith('darwin'):
         osname = 'macos'
         # Take some rather drastic steps to link openssl statically
-        check_call(['brew', 'install', 'openssl', 'pkg-config'])
+        check_call(['brew', 'install', 'openssl', 'xz', 'pkg-config'])
         if platform.machine() == 'x86_64':
             prefix = '/usr/local'
             min_macos_version = '10.11'
         elif platform.machine() == 'arm64':
             prefix = '/opt/homebrew'
             min_macos_version = '11.0'
 
-        osname += '-' + platform.machine()  # Append '-x86_64' or '-arm64' depending on current arch. (TODO: Do this for Linux too, move this below?)
+        # Append '-x86_64' or '-arm64' depending on current arch. (TODO: Do
+        # this for Linux too, move this below?)
+        osname += '-' + platform.machine()
 
         try:
             os.remove(os.path.join(prefix, 'opt', 'openssl', 'lib', 'libssl.dylib'))
             os.remove(os.path.join(prefix, 'opt', 'openssl', 'lib', 'libcrypto.dylib'))
+            os.remove(os.path.join(prefix, 'opt', 'xz', 'lib', 'liblzma.dylib'))
         except Exception:
             pass
         os.environ['PKG_CONFIG_PATH'] = os.path.join(prefix, 'opt', 'openssl', 'lib', 'pkgconfig')
@@ -129,11 +132,18 @@ def build_python():
       check_call(['git', 'clone', 'https://github.com/python/cpython'])
     check_call(['git', 'checkout', 'v' + version], cwd=src_dir)
 
-    min_macos_version_line = '-mmacosx-version-min=' + min_macos_version  # Specify the min OS version we want the build to work on
-    build_flags = min_macos_version_line + ' -Werror=partial-availability'  # Build against latest SDK, but issue an error if using any API that would not work on the min OS version
-    env = os.environ.copy()
-    env['MACOSX_DEPLOYMENT_TARGET'] = min_macos_version
-    check_call(['./configure', 'CFLAGS=' + build_flags, 'CXXFLAGS=' + build_flags, 'LDFLAGS=' + min_macos_version_line], cwd=src_dir, env=env)
+    env = os.environ
+    if sys.platform.startswith('darwin'):
+      # Specify the min OS version we want the build to work on
+      min_macos_version_line = '-mmacosx-version-min=' + min_macos_version
+      build_flags = min_macos_version_line + ' -Werror=partial-availability'
+      # Build against latest SDK, but issue an error if using any API that would not work on the min OS version
+      env = env.copy()
+      env['MACOSX_DEPLOYMENT_TARGET'] = min_macos_version
+      configure_args = ['CFLAGS=' + build_flags, 'CXXFLAGS=' + build_flags, 'LDFLAGS=' + min_macos_version_line]
+    else:
+      configure_args = []
+    check_call(['./configure'] + configure_args, cwd=src_dir, env=env)
     check_call(['make', '-j', str(multiprocessing.cpu_count())], cwd=src_dir, env=env)
     check_call(['make', 'install', 'DESTDIR=install'], cwd=src_dir, env=env)
 
@@ -143,6 +153,7 @@ def build_python():
     # SSL certificates are available (certifi in installed and used by requests).
     pybin = os.path.join(src_dir, 'install', 'usr', 'local', 'bin', 'python3')
     pip = os.path.join(src_dir, 'install', 'usr', 'local', 'bin', 'pip3')
+    check_call([pybin, '-m', 'ensurepip', '--upgrade'])
     check_call([pybin, pip, 'install', 'requests'])
 
     dirname = 'python-%s-%s' % (version, revision)
@@ -154,7 +165,7 @@ def build_python():
     shutil.rmtree(os.path.join(dirname, 'lib', 'python' + major_minor_version, 'test'))
     shutil.rmtree(os.path.join(dirname, 'include'))
     for lib in glob.glob(os.path.join(dirname, 'lib', 'lib*.a')):
-      os.remove(lib)
+        os.remove(lib)
     check_call(['tar', 'zcvf', tarball, dirname])
     print('Uploading: ' + upload_base + tarball)
     check_call(['gsutil', 'cp', '-n', tarball, upload_base + tarball])