diff --git a/c_src/pythonx/python.cpp b/c_src/pythonx/python.cpp index ebf5d27..9a65c3c 100644 --- a/c_src/pythonx/python.cpp +++ b/c_src/pythonx/python.cpp @@ -77,6 +77,7 @@ DEF_SYMBOL(Py_IsFalse) DEF_SYMBOL(Py_IsNone) DEF_SYMBOL(Py_IsTrue) DEF_SYMBOL(Py_SetPythonHome) +DEF_SYMBOL(Py_SetProgramName) dl::LibraryHandle python_library; @@ -150,6 +151,7 @@ void load_python_library(std::string path) { LOAD_SYMBOL(python_library, Py_IsNone) LOAD_SYMBOL(python_library, Py_IsTrue) LOAD_SYMBOL(python_library, Py_SetPythonHome) + LOAD_SYMBOL(python_library, Py_SetProgramName) } void unload_python_library() { diff --git a/c_src/pythonx/python.hpp b/c_src/pythonx/python.hpp index 1bcd14f..20d1414 100644 --- a/c_src/pythonx/python.hpp +++ b/c_src/pythonx/python.hpp @@ -131,6 +131,7 @@ extern int (*Py_IsFalse)(PyObjectPtr); extern int (*Py_IsNone)(PyObjectPtr); extern int (*Py_IsTrue)(PyObjectPtr); extern void (*Py_SetPythonHome)(const wchar_t *); +extern void (*Py_SetProgramName)(const wchar_t *); // Opens Python dynamic library at the given path and looks up all // relevant symbols. diff --git a/c_src/pythonx/pythonx.cpp b/c_src/pythonx/pythonx.cpp index 59f6e06..44d38c5 100644 --- a/c_src/pythonx/pythonx.cpp +++ b/c_src/pythonx/pythonx.cpp @@ -23,6 +23,7 @@ using namespace python; std::mutex init_mutex; bool is_initialized = false; std::wstring python_home_path_w; +std::wstring python_executable_path_w; std::map> compilation_cache; std::mutex compilation_cache_mutex; @@ -225,6 +226,7 @@ ERL_NIF_TERM py_object_to_binary_term(ErlNifEnv *env, PyObjectPtr py_object) { fine::Ok<> init(ErlNifEnv *env, std::string python_dl_path, ErlNifBinary python_home_path, + ErlNifBinary python_executable_path, std::vector sys_paths) { auto init_guard = std::lock_guard(init_mutex); @@ -240,24 +242,36 @@ fine::Ok<> init(ErlNifEnv *env, std::string python_dl_path, python_home_path_w = std::wstring( python_home_path.data, python_home_path.data + python_home_path.size); - // As part of the initialization, sys.path is set. It is important + python_executable_path_w = + std::wstring(python_executable_path.data, + python_executable_path.data + python_executable_path.size); + + // As part of the initialization, sys.path gets set. It is important // that it gets set correctly, so that the built-in modules can be // found, otherwise the initialization fails. This logic is internal // to Python, but we can configure base paths used to infer sys.path. - // The Limited API exposes Py_SetPythonHome and Py_SetProgramName. - // Technically we could use either of them, while Py_SetProgramName - // has the advantage that, when set to the executable inside venv, - // it results in the packages directory being added to sys.path - // automatically. However, when testing Py_SetProgramName did not - // work as expected in Python 3.10 on Windows. For this reason we - // use Py_SetPythonHome, which seems more reliable, and add other - // paths to sys.path manually. + // The Limited API exposes Py_SetPythonHome and Py_SetProgramName and + // it appears that setting either of them alone should be sufficient. + // + // Py_SetProgramName has the advantage that, when set to the executable + // inside venv, it results in the packages directory being added to + // sys.path automatically, however, when tested, this did not work + // as expected in Python 3.10 on Windows. For this reason we prefer + // to use Py_SetPythonHome and add other paths to sys.path manually. + // + // Even then, we still want to set Py_SetProgramName to a Python + // executable, otherwise `sys.executable` is going to point to the + // BEAM executable (`argv[0]`), which can be problematic. + // + // In the end, the most reliable combination seems to be to set both, + // and also add the extra sys.path manually. // // Note that Python home is the directory with lib/ child directory // containing the built-in Python modules [1]. // // [1]: https://docs.python.org/3/using/cmdline.html#envvar-PYTHONHOME Py_SetPythonHome(python_home_path_w.c_str()); + Py_SetProgramName(python_executable_path_w.c_str()); Py_InitializeEx(0); diff --git a/lib/pythonx.ex b/lib/pythonx.ex index 0db8f36..5513791 100644 --- a/lib/pythonx.ex +++ b/lib/pythonx.ex @@ -12,47 +12,6 @@ defmodule Pythonx do @type encoder :: (term(), encoder() -> Object.t()) - @doc """ - Initializes the Python interpreter. - - > #### Reproducability {: .info} - > - > This function can be called to use a custom Python installation, - > however in most cases it is more convenient to call `uv_init/2`, - > which installs Python and dependencies, and then automatically - > initializes the interpreter using the correct paths. - - The `python_dl_path` argument is the Python dynamically linked - library file. The usual file name is `libpython3.x.so` (Linux), - `libpython3.x.dylib` (macOS), `python3x.dll` (Windows). - - The `python_home_path` is the Python home directory, where the Python - built-in modules reside. Specifically, the modules should be located - in `{python_home_path}/lib/pythonx.y` (Linux and macOS) or - `{python_home_path}/Lib` (Windows). - - ## Options - - * `:sys_paths` - directories to be added to the module search path - (`sys.path`). Defaults to `[]`. - - """ - @spec init(String.t(), String.t(), keyword()) :: :ok - def init(python_dl_path, python_home_path, opts \\ []) - when is_binary(python_dl_path) and is_binary(python_home_path) and is_list(opts) do - opts = Keyword.validate!(opts, sys_paths: []) - - if not File.exists?(python_dl_path) do - raise ArgumentError, "the given dynamic library file does not exist: #{python_dl_path}" - end - - if not File.dir?(python_home_path) do - raise ArgumentError, "the given python home directory does not exist: #{python_home_path}" - end - - Pythonx.NIF.init(python_dl_path, python_home_path, opts[:sys_paths]) - end - @doc ~S''' Installs Python and dependencies using [uv](https://docs.astral.sh/uv) package manager and initializes the interpreter. @@ -99,6 +58,53 @@ defmodule Pythonx do Pythonx.Uv.init(pyproject_toml, false) end + # Initializes the Python interpreter. + # + # > #### Reproducability {: .info} + # > + # > This function can be called to use a custom Python installation, + # > however in most cases it is more convenient to call `uv_init/2`, + # > which installs Python and dependencies, and then automatically + # > initializes the interpreter using the correct paths. + # + # `python_dl_path` is the Python dynamically linked library file. + # The usual file name is `libpython3.x.so` (Linux), `libpython3.x.dylib` + # (macOS), `python3x.dll` (Windows). + # + # `python_home_path` is the Python home directory, where the Python + # built-in modules reside. Specifically, the modules should be + # located in `{python_home_path}/lib/pythonx.y` (Linux and macOS) + # or `{python_home_path}/Lib` (Windows). + # + # `python_executable_path` is the Python executable file. + # + # ## Options + # + # * `:sys_paths` - directories to be added to the module search path + # (`sys.path`). Defaults to `[]`. + # + @doc false + @spec init(String.t(), String.t(), keyword()) :: :ok + def init(python_dl_path, python_home_path, python_executable_path, opts \\ []) + when is_binary(python_dl_path) and is_binary(python_home_path) + when is_binary(python_executable_path) and is_list(opts) do + opts = Keyword.validate!(opts, sys_paths: []) + + if not File.exists?(python_dl_path) do + raise ArgumentError, "the given dynamic library file does not exist: #{python_dl_path}" + end + + if not File.dir?(python_home_path) do + raise ArgumentError, "the given python home directory does not exist: #{python_home_path}" + end + + if not File.exists?(python_home_path) do + raise ArgumentError, "the given python executable does not exist: #{python_executable_path}" + end + + Pythonx.NIF.init(python_dl_path, python_home_path, python_executable_path, opts[:sys_paths]) + end + @doc ~S''' Evaluates the Python `code`. diff --git a/lib/pythonx/nif.ex b/lib/pythonx/nif.ex index 47a4256..774f314 100644 --- a/lib/pythonx/nif.ex +++ b/lib/pythonx/nif.ex @@ -12,7 +12,7 @@ defmodule Pythonx.NIF do end end - def init(_python_dl_path, _python_home_path, _sys_paths), do: err!() + def init(_python_dl_path, _python_home_path, _python_executable_path, _sys_paths), do: err!() def terminate(), do: err!() def janitor_decref(_ptr), do: err!() def none_new(), do: err!() diff --git a/lib/pythonx/uv.ex b/lib/pythonx/uv.ex index bbae5f1..41f3a04 100644 --- a/lib/pythonx/uv.ex +++ b/lib/pythonx/uv.ex @@ -106,12 +106,19 @@ defmodule Pythonx.Uv do python_home_path = make_windows_slashes(root_dir) + python_executable_path = + abs_executable_dir + |> Path.join("python.exe") + |> make_windows_slashes() + venv_packages_path = project_dir |> Path.join(".venv/Lib/site-packages") |> make_windows_slashes() - Pythonx.init(python_dl_path, python_home_path, sys_paths: [venv_packages_path]) + Pythonx.init(python_dl_path, python_home_path, python_executable_path, + sys_paths: [venv_packages_path] + ) {:unix, osname} -> dl_extension = @@ -128,12 +135,16 @@ defmodule Pythonx.Uv do python_home_path = root_dir + python_executable_path = Path.join(abs_executable_dir, "python") + venv_packages_path = project_dir |> Path.join(".venv/lib/python3*/site-packages") |> wildcard_one!() - Pythonx.init(python_dl_path, python_home_path, sys_paths: [venv_packages_path]) + Pythonx.init(python_dl_path, python_home_path, python_executable_path, + sys_paths: [venv_packages_path] + ) end end