Update init so that sys.executable points to python (#4)

Author: jonatanklosko

c_src/pythonx/python.cpp

@@ -77,6 +77,7 @@ DEF_SYMBOL(Py_IsFalse)
 DEF_SYMBOL(Py_IsNone)
 DEF_SYMBOL(Py_IsTrue)
 DEF_SYMBOL(Py_SetPythonHome)
+DEF_SYMBOL(Py_SetProgramName)
 
 dl::LibraryHandle python_library;
 
@@ -150,6 +151,7 @@ void load_python_library(std::string path) {
   LOAD_SYMBOL(python_library, Py_IsNone)
   LOAD_SYMBOL(python_library, Py_IsTrue)
   LOAD_SYMBOL(python_library, Py_SetPythonHome)
+  LOAD_SYMBOL(python_library, Py_SetProgramName)
 }
 
 void unload_python_library() {

c_src/pythonx/python.hpp

@@ -131,6 +131,7 @@ extern int (*Py_IsFalse)(PyObjectPtr);
 extern int (*Py_IsNone)(PyObjectPtr);
 extern int (*Py_IsTrue)(PyObjectPtr);
 extern void (*Py_SetPythonHome)(const wchar_t *);
+extern void (*Py_SetProgramName)(const wchar_t *);
 
 // Opens Python dynamic library at the given path and looks up all
 // relevant symbols.

c_src/pythonx/pythonx.cpp

@@ -23,6 +23,7 @@ using namespace python;
 std::mutex init_mutex;
 bool is_initialized = false;
 std::wstring python_home_path_w;
+std::wstring python_executable_path_w;
 std::map<std::string, std::tuple<PyObjectPtr, PyObjectPtr>> compilation_cache;
 std::mutex compilation_cache_mutex;
 
@@ -225,6 +226,7 @@ ERL_NIF_TERM py_object_to_binary_term(ErlNifEnv *env, PyObjectPtr py_object) {
 
 fine::Ok<> init(ErlNifEnv *env, std::string python_dl_path,
                 ErlNifBinary python_home_path,
+                ErlNifBinary python_executable_path,
                 std::vector<ErlNifBinary> sys_paths) {
   auto init_guard = std::lock_guard<std::mutex>(init_mutex);
 
@@ -240,24 +242,36 @@ fine::Ok<> init(ErlNifEnv *env, std::string python_dl_path,
   python_home_path_w = std::wstring(
       python_home_path.data, python_home_path.data + python_home_path.size);
 
-  // As part of the initialization, sys.path is set. It is important
+  python_executable_path_w =
+      std::wstring(python_executable_path.data,
+                   python_executable_path.data + python_executable_path.size);
+
+  // As part of the initialization, sys.path gets set. It is important
   // that it gets set correctly, so that the built-in modules can be
   // found, otherwise the initialization fails. This logic is internal
   // to Python, but we can configure base paths used to infer sys.path.
-  // The Limited API exposes Py_SetPythonHome and Py_SetProgramName.
-  // Technically we could use either of them, while Py_SetProgramName
-  // has the advantage that, when set to the executable inside venv,
-  // it results in the packages directory being added to sys.path
-  // automatically. However, when testing Py_SetProgramName did not
-  // work as expected in Python 3.10 on Windows. For this reason we
-  // use Py_SetPythonHome, which seems more reliable, and add other
-  // paths to sys.path manually.
+  // The Limited API exposes Py_SetPythonHome and Py_SetProgramName and
+  // it appears that setting either of them alone should be sufficient.
+  //
+  // Py_SetProgramName has the advantage that, when set to the executable
+  // inside venv, it results in the packages directory being added to
+  // sys.path automatically, however, when tested, this did not work
+  // as expected in Python 3.10 on Windows. For this reason we prefer
+  // to use Py_SetPythonHome and add other paths to sys.path manually.
+  //
+  // Even then, we still want to set Py_SetProgramName to a Python
+  // executable, otherwise `sys.executable` is going to point to the
+  // BEAM executable (`argv[0]`), which can be problematic.
+  //
+  // In the end, the most reliable combination seems to be to set both,
+  // and also add the extra sys.path manually.
   //
   // Note that Python home is the directory with lib/ child directory
   // containing the built-in Python modules [1].
   //
   // [1]: https://docs.python.org/3/using/cmdline.html#envvar-PYTHONHOME
   Py_SetPythonHome(python_home_path_w.c_str());
+  Py_SetProgramName(python_executable_path_w.c_str());
 
   Py_InitializeEx(0);
 

lib/pythonx.ex

@@ -12,47 +12,6 @@ defmodule Pythonx do
 
   @type encoder :: (term(), encoder() -> Object.t())
 
-  @doc """
-  Initializes the Python interpreter.
-
-  > #### Reproducability {: .info}
-  >
-  > This function can be called to use a custom Python installation,
-  > however in most cases it is more convenient to call `uv_init/2`,
-  > which installs Python and dependencies, and then automatically
-  > initializes the interpreter using the correct paths.
-
-  The `python_dl_path` argument is the Python dynamically linked
-  library file. The usual file name is `libpython3.x.so` (Linux),
-  `libpython3.x.dylib` (macOS), `python3x.dll` (Windows).
-
-  The `python_home_path` is the Python home directory, where the Python
-  built-in modules reside. Specifically, the modules should be located
-  in `{python_home_path}/lib/pythonx.y` (Linux and macOS) or
-  `{python_home_path}/Lib` (Windows).
-
-  ## Options
-
-    * `:sys_paths` - directories to be added to the module search path
-      (`sys.path`). Defaults to `[]`.
-
-  """
-  @spec init(String.t(), String.t(), keyword()) :: :ok
-  def init(python_dl_path, python_home_path, opts \\ [])
-      when is_binary(python_dl_path) and is_binary(python_home_path) and is_list(opts) do
-    opts = Keyword.validate!(opts, sys_paths: [])
-
-    if not File.exists?(python_dl_path) do
-      raise ArgumentError, "the given dynamic library file does not exist: #{python_dl_path}"
-    end
-
-    if not File.dir?(python_home_path) do
-      raise ArgumentError, "the given python home directory does not exist: #{python_home_path}"
-    end
-
-    Pythonx.NIF.init(python_dl_path, python_home_path, opts[:sys_paths])
-  end
-
   @doc ~S'''
   Installs Python and dependencies using [uv](https://docs.astral.sh/uv)
   package manager and initializes the interpreter.
@@ -99,6 +58,53 @@ defmodule Pythonx do
     Pythonx.Uv.init(pyproject_toml, false)
   end
 
+  # Initializes the Python interpreter.
+  #
+  # > #### Reproducability {: .info}
+  # >
+  # > This function can be called to use a custom Python installation,
+  # > however in most cases it is more convenient to call `uv_init/2`,
+  # > which installs Python and dependencies, and then automatically
+  # > initializes the interpreter using the correct paths.
+  #
+  # `python_dl_path` is the Python dynamically linked library file.
+  # The usual file name is `libpython3.x.so` (Linux), `libpython3.x.dylib`
+  # (macOS), `python3x.dll` (Windows).
+  #
+  # `python_home_path` is the Python home directory, where the Python
+  # built-in modules reside. Specifically, the modules should be
+  # located in `{python_home_path}/lib/pythonx.y` (Linux and macOS)
+  # or `{python_home_path}/Lib` (Windows).
+  #
+  # `python_executable_path` is the Python executable file.
+  #
+  # ## Options
+  #
+  #   * `:sys_paths` - directories to be added to the module search path
+  #     (`sys.path`). Defaults to `[]`.
+  #
+  @doc false
+  @spec init(String.t(), String.t(), keyword()) :: :ok
+  def init(python_dl_path, python_home_path, python_executable_path, opts \\ [])
+      when is_binary(python_dl_path) and is_binary(python_home_path)
+      when is_binary(python_executable_path) and is_list(opts) do
+    opts = Keyword.validate!(opts, sys_paths: [])
+
+    if not File.exists?(python_dl_path) do
+      raise ArgumentError, "the given dynamic library file does not exist: #{python_dl_path}"
+    end
+
+    if not File.dir?(python_home_path) do
+      raise ArgumentError, "the given python home directory does not exist: #{python_home_path}"
+    end
+
+    if not File.exists?(python_home_path) do
+      raise ArgumentError, "the given python executable does not exist: #{python_executable_path}"
+    end
+
+    Pythonx.NIF.init(python_dl_path, python_home_path, python_executable_path, opts[:sys_paths])
+  end
+
   @doc ~S'''
   Evaluates the Python `code`.
 

lib/pythonx/nif.ex

@@ -12,7 +12,7 @@ defmodule Pythonx.NIF do
     end
   end
 
-  def init(_python_dl_path, _python_home_path, _sys_paths), do: err!()
+  def init(_python_dl_path, _python_home_path, _python_executable_path, _sys_paths), do: err!()
   def terminate(), do: err!()
   def janitor_decref(_ptr), do: err!()
   def none_new(), do: err!()

lib/pythonx/uv.ex

@@ -106,12 +106,19 @@ defmodule Pythonx.Uv do
 
         python_home_path = make_windows_slashes(root_dir)
 
+        python_executable_path =
+          abs_executable_dir
+          |> Path.join("python.exe")
+          |> make_windows_slashes()
+
         venv_packages_path =
           project_dir
           |> Path.join(".venv/Lib/site-packages")
           |> make_windows_slashes()
 
-        Pythonx.init(python_dl_path, python_home_path, sys_paths: [venv_packages_path])
+        Pythonx.init(python_dl_path, python_home_path, python_executable_path,
+          sys_paths: [venv_packages_path]
+        )
 
       {:unix, osname} ->
         dl_extension =
@@ -128,12 +135,16 @@ defmodule Pythonx.Uv do
 
         python_home_path = root_dir
 
+        python_executable_path = Path.join(abs_executable_dir, "python")
+
         venv_packages_path =
           project_dir
           |> Path.join(".venv/lib/python3*/site-packages")
           |> wildcard_one!()
 
-        Pythonx.init(python_dl_path, python_home_path, sys_paths: [venv_packages_path])
+        Pythonx.init(python_dl_path, python_home_path, python_executable_path,
+          sys_paths: [venv_packages_path]
+        )
     end
   end