ginstall: add scipy-1.9.3.patch

cosminbasca · timfel · commit cba306e7a72e · 2023-03-29T09:45:05.000+02:00
- scipy 1.9.x - setup env to build with flang and meson
- scipy-1.9.3.patch - cythonize in process
diff --git a/graalpython/lib-graalpython/modules/ginstall.py b/graalpython/lib-graalpython/modules/ginstall.py
@@ -84,6 +84,26 @@ def get_module_name(package_name):
     return module_name.replace('-', '_')
 
 
+def have_flang_new():
+    env_path = os.environ.get('PATH', '').split(os.pathsep)
+    for p in env_path:
+        flang_new = os.path.join(p, 'flang-new')
+        if os.path.isfile(flang_new):
+            return True
+    return False
+
+
+def get_flang_new_lib_dir():
+    try:
+        output = subprocess.check_output(['flang-new', '--version'])
+    except (OSError, subprocess.CalledProcessError):
+        pass
+    else:
+        flang_dir = output.splitlines()[-1].split()[-1].strip().decode("utf-8")
+        return os.path.normpath(os.path.join(flang_dir, '..', 'lib'))
+    return None
+
+
 def get_path_env_var(var):
     env_var = os.environ.get(var, None)
     if isinstance(env_var, str) and env_var.lower() == 'none':
@@ -406,10 +426,13 @@ def make_site_cfg(root):
             append_env_var(numpy_build_env, 'CFLAGS', '-Wno-error=implicit-function-declaration')
             info(f"have lapack or blas ... CLFAGS={numpy_build_env['CFLAGS']}")
 
-        numpy_build_env["FC"] = "flang-new"
-        install_from_pypi("numpy==1.23.5", build_cmd=["build_ext", "--disable-optimization",
-                                                      "config", "--fcompiler=flang-new"], env=numpy_build_env,
-                          pre_install_hook=make_site_cfg, **kwargs)
+        with_flang = have_flang_new()
+        build_cmd = ["build_ext", "--disable-optimization"]
+        if with_flang:
+            numpy_build_env["FC"] = "flang-new"
+            build_cmd += ["config", "--fcompiler=flang-new"]
+        install_from_pypi("numpy==1.23.5", build_cmd=build_cmd, env=numpy_build_env, pre_install_hook=make_site_cfg,
+                          **kwargs)
 
         # print numpy configuration
         if sys.implementation.name != 'graalpy' or __graalpython__.platform_id != 'managed':
@@ -491,7 +514,6 @@ def scipy(**kwargs):
                 xit("SciPy can only be installed within a venv.")
             from distutils.sysconfig import get_config_var
             scipy_build_env["LDFLAGS"] = get_config_var("LDFLAGS")
-            scipy_build_env["FC"] = "flang-new"
 
         if have_lapack() or have_openblas():
             append_env_var(scipy_build_env, 'CFLAGS', '-Wno-error=implicit-function-declaration')
@@ -502,8 +524,34 @@ def scipy(**kwargs):
         pybind11(**kwargs)
         pythran(**kwargs)
 
-        install_from_pypi("scipy==1.8.1", build_cmd=["config", "--fcompiler=flang-new"],
-                          env=scipy_build_env, **kwargs)
+        scipy_version = "1.8.1"
+        # scipy_version = "1.9.3"
+        with_meson = False
+        if scipy_version >= "1.9.1":
+            meson(**kwargs)
+            ninja(**kwargs)
+            with_flang = have_flang_new()
+            with_meson = True
+            if with_flang:
+                # until flang-new can compile propack we disable it (during runtime)
+                scipy_build_env["USE_PROPACK"] = "0"
+                scipy_build_env['CC'] = 'clang'
+                scipy_build_env['CXX'] = 'clang++'
+                scipy_build_env['FC'] = 'flang-new'
+                if sys.implementation.name == "graalpy":
+                    cflags = "-flto=full"
+                else:
+                    ld = 'lld'
+                    cflags = "-flto=full -fuse-ld=lld -Wl,--mllvm=-lto-embed-bitcode=optimized,--lto-O0"
+                    scipy_build_env['CC_LD'] = ld
+                    scipy_build_env['CXX_LD'] = ld
+                    scipy_build_env['FC_LD'] = ld
+                scipy_build_env['CFLAGS'] = cflags
+                scipy_build_env['CXXFLAGS'] = cflags
+                scipy_build_env['FFLAGS'] = cflags
+                scipy_build_env['FFLAGS'] = cflags
+                append_env_var(scipy_build_env, 'LDFLAGS', f'-L{get_flang_new_lib_dir()}')
+        install_from_pypi(f"scipy=={scipy_version}", env=scipy_build_env, with_meson=with_meson, **kwargs)
 
     @pip_package()
     def scikit_learn(**kwargs):
@@ -631,7 +679,7 @@ def _download_with_curl_and_extract(dest_dir, url, quiet=False):
 
 
 def _install_from_url(url, package, extra_opts=None, add_cflags="", ignore_errors=False, env=None, version=None,
-                      pre_install_hook=None, build_cmd=None, debug_build=False):
+                      pre_install_hook=None, build_cmd=None, debug_build=False, with_meson=False):
     if build_cmd is None:
         build_cmd = []
     if env is None:
@@ -683,24 +731,36 @@ def _install_from_url(url, package, extra_opts=None, add_cflags="", ignore_error
         os.path.join(tempdir, bare_name, subdir)
         run_cmd(["patch", "-d", os.path.join(tempdir, bare_name, subdir), "-p1", "-i", patch_file_path], quiet=quiet)
 
+    tmp_cwd = os.path.join(tempdir, bare_name)
     if pre_install_hook:
-        pre_install_hook(os.path.join(tempdir, bare_name))
+        pre_install_hook(tmp_cwd)
 
     if "--user" not in extra_opts and "--prefix" not in extra_opts and site.ENABLE_USER_SITE:
         user_arg = ["--user"]
     else:
         user_arg = []
+
     start = time.time()
-    cmd = [sys.executable]
-    if debug_build and sys.implementation.name == 'graalpy':
-        cmd += ["-debug-java", "--python.ExposeInternalSources", "--python.WithJavaStacktrace=2"]
-    cmd += ["setup.py"] + build_cmd + ["install"] + user_arg + extra_opts
-    status = run_cmd(cmd, env=setup_env,
-                     cwd=os.path.join(tempdir, bare_name), quiet=quiet)
+    if with_meson:
+        for cmd in [
+            ['meson', 'setup', f'--prefix={os.environ.get("VIRTUAL_ENV")}', 'build'],
+            ['meson', 'compile', '-C', 'build', '--ninja-args=-j4'],  # limit concurrency to 4
+            ['meson', 'install', '-C', 'build'],
+        ]:
+            status = run_cmd(cmd, env=setup_env, cwd=tmp_cwd, quiet=quiet)
+            if status != 0 and not ignore_errors:
+                xit(f"An error occurred trying to run `{' '.join(cmd)}'")
+    else:
+        cmd = [sys.executable]
+        if debug_build and sys.implementation.name == 'graalpy':
+            cmd += ["-debug-java", "--python.ExposeInternalSources", "--python.WithJavaStacktrace=2"]
+        cmd += ["setup.py"] + build_cmd + ["install"] + user_arg + extra_opts
+        status = run_cmd(cmd, env=setup_env, cwd=tmp_cwd, quiet=quiet)
+        if status != 0 and not ignore_errors:
+            xit("An error occurred trying to run `setup.py install {!s} {}'", user_arg, " ".join(extra_opts))
+
     end = time.time()
-    if status != 0 and not ignore_errors:
-        xit("An error occurred trying to run `setup.py install {!s} {}'", user_arg, " ".join(extra_opts))
-    elif quiet:
+    if quiet:
         info("{} successfully installed (took {:.2f} s)", package, (end - start))
 
 
@@ -754,7 +814,7 @@ def package_from_path(pth):
 
 
 def install_from_pypi(package, extra_opts=None, add_cflags="", ignore_errors=True, env=None, pre_install_hook=None,
-                      build_cmd=None, debug_build=False):
+                      build_cmd=None, debug_build=False, with_meson=False):
     if build_cmd is None:
         build_cmd = []
     if extra_opts is None:
@@ -810,7 +870,7 @@ def set_if_exists(env_var, conf_var):
     if url:
         _install_from_url(url, package=package, extra_opts=extra_opts, add_cflags=add_cflags,
                           ignore_errors=ignore_errors, env=env, version=version, pre_install_hook=pre_install_hook,
-                          build_cmd=build_cmd, debug_build=debug_build)
+                          build_cmd=build_cmd, debug_build=debug_build, with_meson=with_meson)
     else:
         xit("Package not found: '{!s}'", package)
 
diff --git a/graalpython/lib-graalpython/patches/scipy/sdist/scipy-1.9.3.patch b/graalpython/lib-graalpython/patches/scipy/sdist/scipy-1.9.3.patch
@@ -0,0 +1,120 @@
+diff --git a/meson.build b/meson.build
+index 5b4a1d52..c29ec297 100644
+--- a/meson.build
++++ b/meson.build
+@@ -11,7 +11,7 @@ project(
+     'buildtype=debugoptimized',
+     'c_std=c99',
+     'cpp_std=c++14',
+-    'fortran_std=legacy',
++    'fortran_std=none',
+     'blas=openblas',
+     'lapack=openblas'
+   ],
+diff --git a/scipy/sparse/linalg/meson.build b/scipy/sparse/linalg/meson.build
+index db2f199d..763e1ef6 100644
+--- a/scipy/sparse/linalg/meson.build
++++ b/scipy/sparse/linalg/meson.build
+@@ -16,7 +16,7 @@ py3.install_sources([
+   subdir: 'scipy/sparse/linalg'
+ )
+ 
+-subdir('_propack')
++# subdir('_propack')
+ subdir('_isolve')
+ subdir('_dsolve')
+ subdir('_eigen')
+diff --git a/scipy/special/meson.build b/scipy/special/meson.build
+index 8fe534be..133be12b 100644
+--- a/scipy/special/meson.build
++++ b/scipy/special/meson.build
+@@ -459,24 +459,24 @@ py3.extension_module('_test_round',
+ # Must use `custom_target`, because `py3.install_sources` does not work with
+ # generated sources - see https://github.com/mesonbuild/meson/issues/7372
+ npz_files = [
+-  [
+-    '_data_boost',
+-    'tests/data/boost/acosh_data_ipp/acosh_data.txt',
+-    'boost',
+-    'boost.npz'
+-  ],
+-  [
+-    '_data_gsl',
+-    'tests/data/gsl/mathieu_ab.txt',
+-    'gsl',
+-    'gsl.npz'
+-  ],
+-  [
+-    '_data_local',
+-    'tests/data/local/ellipkm1.txt',
+-    'local',
+-    'local.npz'
+-  ],
++  # [
++  #   '_data_boost',
++  #   'tests/data/boost/acosh_data_ipp/acosh_data.txt',
++  #   'boost',
++  #   'boost.npz'
++  # ],
++  # [
++  #   '_data_gsl',
++  #   'tests/data/gsl/mathieu_ab.txt',
++  #   'gsl',
++  #   'gsl.npz'
++  # ],
++  # [
++  #   '_data_local',
++  #   'tests/data/local/ellipkm1.txt',
++  #   'local',
++  #   'local.npz'
++  # ],
+ ]
+ 
+ foreach npz_file: npz_files
+diff --git a/tools/cythonize.py b/tools/cythonize.py
+index 402824cb..b6aba959 100755
+--- a/tools/cythonize.py
++++ b/tools/cythonize.py
+@@ -95,21 +95,19 @@ def process_pyx(fromfile, tofile, cwd):
+     if tofile.endswith('.cxx'):
+         flags += ['--cplus']
+ 
+-    try:
++    try:    
++        from Cython.Compiler.Main import setuptools_main
++        prev_argv = sys.argv[1:]
+         try:
+-            r = subprocess.call(['cython'] + flags + ["-o", tofile, fromfile], cwd=cwd)
+-            if r != 0:
+-                raise Exception('Cython failed')
+-        except OSError as e:
+-            # There are ways of installing Cython that don't result in a cython
+-            # executable on the path, see gh-2397.
+-            r = subprocess.call([sys.executable, '-c',
+-                                 'import sys; from Cython.Compiler.Main import '
+-                                 'setuptools_main as main; sys.exit(main())'] + flags +
+-                                 ["-o", tofile, fromfile],
+-                                cwd=cwd)
+-            if r != 0:
+-                raise Exception("Cython either isn't installed or it failed.") from e
++            oldcwd = os.getcwd()
++            os.chdir(cwd)
++            sys.argv[1:] = flags + ["-o", tofile, fromfile]
++            retcode = setuptools_main()
++            if retcode:
++                sys.exit(retcode)
++        finally:
++            sys.argv[1:] = prev_argv
++            os.chdir(oldcwd)
+     except OSError as e:
+         raise OSError('Cython needs to be installed') from e
+ 
+@@ -253,7 +251,7 @@ def find_process_files(root_dir):
+     lock = Lock()
+ 
+     try:
+-        num_proc = int(os.environ.get('SCIPY_NUM_CYTHONIZE_JOBS', cpu_count()))
++        num_proc = int(os.environ.get('SCIPY_NUM_CYTHONIZE_JOBS', 1))
+         pool = Pool(processes=num_proc)
+     except ImportError as e:
+         # Allow building (single-threaded) on GNU/Hurd, which does not
