unix: upgrade libedit 20210910-3.1 -> 20240808-3.1 (#466)

indygreg · web-flow · commit 3c4fe233ae1f · 2025-01-06T13:17:09.000-06:00
We were soft blocked on upgrading due to musl compatibility issues.

It looks like these got fixed upstream. So we refreshed the configure
patch and libedit build _just worked_.

However, Python 3.9 and 3.10 encountered compile errors with the
newer version.

On 3.10 we worked around this bug by backporting a patch from 3.11.
On 3.9, the backport was non-trivial, so I just hacked up the existing
3.9 patch to manually change some C preprocessor checks to key off
libedit.

While diffing `Modules/readline.c` I found another patch related to
fixing completer delims. While strictly not required, it was trivial
to backport to 3.10 to fix some missing functionality. So I did.

3.13 initially didn't like the upgraded libedit because
we were manually defining a preprocessor variable (introduced in 3.13
by upstream commit 8515fd79fef1ac16d7848cec5ec1797294cb5366). Removing
the variable and letting configure deduce things with the newer libedit
appears to _just work_. Perhaps upstream configure doesn't implement
the feature detection properly on older libedit versions?
diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh
@@ -221,19 +221,43 @@ if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_9}" ]; then
     patch -p1 -i ${ROOT}/patch-decimal-modern-mpdecimal.patch
 fi
 
+# We build against libedit instead of readline in all environments.
+#
+# On macOS, we use the system/SDK libedit, which is likely somewhat old.
+#
+# On Linux, we use our own libedit, which should be modern.
+#
 # CPython 3.10 added proper support for building against libedit outside of
-# macOS. On older versions, we need to patch readline.c.
-if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_9}" ]; then
+# macOS. On older versions, we need to hack up readline.c to build against
+# libedit. This patch breaks older libedit (as seen on macOS) so don't apply
+# on macOS.
+if [[ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_9}" && "${PYBUILD_PLATFORM}" != "macos" ]]; then
     # readline.c assumes that a modern readline API version has a free_history_entry().
     # but libedit does not. Change the #ifdef accordingly.
     #
     # Similarly, we invoke configure using readline, which sets
     # HAVE_RL_COMPLETION_SUPPRESS_APPEND improperly. So hack that. This is a bug
     # in our build system, as we should probably be invoking configure again when
     # using libedit.
+    #
+    # Similar workaround for on_completion_display_matches_hook.
     patch -p1 -i ${ROOT}/patch-readline-libedit.patch
 fi
 
+if [ "${PYTHON_MAJMIN_VERSION}" = "3.10" ]; then
+    # Even though 3.10 is libedit aware, it isn't compatible with newer
+    # versions of libedit. We need to backport a 3.11 patch to teach the
+    # build system about completions.
+    # Backport of 9e9df93ffc6df5141843caf651d33d446676a414 from 3.11.
+    patch -p1 -i ${ROOT}/patch-readline-libedit-completions.patch
+
+    # 3.11 has a patch related to completer delims that closes a feature
+    # gap. Backport it as a quality of life enhancement.
+    #
+    # Backport of 42dd2613fe4bc61e1f633078560f2d84a0a16c3f from 3.11.
+    patch -p1 -i ${ROOT}/patch-readline-libedit-completer-delims.patch
+fi
+
 # iOS doesn't have system(). Teach posixmodule.c about that.
 # Python 3.11 makes this a configure time check, so we don't need the patch there.
 if [[ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_10}" ]]; then
diff --git a/cpython-unix/build-libedit.sh b/cpython-unix/build-libedit.sh
@@ -19,13 +19,13 @@ pushd libedit-${LIBEDIT_VERSION}
 # run-time. So we hack up the configure script instead.
 patch -p1 << "EOF"
 diff --git a/configure b/configure
-index 26dd8d0..4b6d47c 100755
+index 614795f..4671f1b 100755
 --- a/configure
 +++ b/configure
-@@ -12921,14 +12921,14 @@ test -n "$NROFF" || NROFF="/bin/false"
- 
- 
- 
+@@ -14154,14 +14154,14 @@ test -n "$NROFF" || NROFF="/bin/false"
+
+
+
 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for tgetent in -lncurses" >&5
 -printf %s "checking for tgetent in -lncurses... " >&6; }
 -if test ${ac_cv_lib_ncurses_tgetent+y}
@@ -34,26 +34,28 @@ index 26dd8d0..4b6d47c 100755
 +if test ${ac_cv_lib_ncursesw_tgetent+y}
  then :
    printf %s "(cached) " >&6
- else $as_nop
-   ac_check_lib_save_LIBS=$LIBS
+ else case e in #(
+   e) ac_check_lib_save_LIBS=$LIBS
 -LIBS="-lncurses  $LIBS"
 +LIBS="-lncursesw  $LIBS"
  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
  /* end confdefs.h.  */
- 
-@@ -12946,21 +12946,21 @@ return tgetent ();
+
+@@ -14185,9 +14185,9 @@ return tgetent ();
  _ACEOF
  if ac_fn_c_try_link "$LINENO"
  then :
 -  ac_cv_lib_ncurses_tgetent=yes
 +  ac_cv_lib_ncursesw_tgetent=yes
- else $as_nop
--  ac_cv_lib_ncurses_tgetent=no
-+  ac_cv_lib_ncursesw_tgetent=no
+ else case e in #(
+-  e) ac_cv_lib_ncurses_tgetent=no ;;
++  e) ac_cv_lib_ncursesw_tgetent=no ;;
+ esac
  fi
  rm -f core conftest.err conftest.$ac_objext conftest.beam \
-     conftest$ac_exeext conftest.$ac_ext
- LIBS=$ac_check_lib_save_LIBS
+@@ -14195,13 +14195,13 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \
+ LIBS=$ac_check_lib_save_LIBS ;;
+ esac
  fi
 -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ncurses_tgetent" >&5
 -printf "%s\n" "$ac_cv_lib_ncurses_tgetent" >&6; }
@@ -63,21 +65,21 @@ index 26dd8d0..4b6d47c 100755
 +if test "x$ac_cv_lib_ncursesw_tgetent" = xyes
  then :
    printf "%s\n" "#define HAVE_LIBNCURSES 1" >>confdefs.h
- 
+
 -  LIBS="-lncurses $LIBS"
 +  LIBS="-lncursesw $LIBS"
- 
- else $as_nop
-   { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for tgetent in -lcurses" >&5
-@@ -13089,7 +13089,7 @@ then :
+
+ else case e in #(
+   e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for tgetent in -lcurses" >&5
+@@ -14354,7 +14354,7 @@ then :
    LIBS="-ltinfo $LIBS"
- 
- else $as_nop
--  as_fn_error $? "libncurses, libcurses, libtermcap or libtinfo is required!" "$LINENO" 5
-+  as_fn_error $? "libncursesw, libcurses, libtermcap or libtinfo is required!" "$LINENO" 5
- 
+
+ else case e in #(
+-  e) as_fn_error $? "libncurses, libcurses, libtermcap or libtinfo is required!" "$LINENO" 5
++  e) as_fn_error $? "libncursesw, libcurses, libtermcap or libtinfo is required!" "$LINENO" 5
+        ;;
+ esac
  fi
- 
 EOF
 
 cflags="${EXTRA_TARGET_CFLAGS} -fPIC -I${TOOLS_PATH}/deps/include -I${TOOLS_PATH}/deps/include/ncursesw"
diff --git a/cpython-unix/extension-modules.yml b/cpython-unix/extension-modules.yml
@@ -880,8 +880,6 @@ readline:
     - readline.c
   defines:
     - USE_LIBEDIT=1
-    # While some versions do not, our readline `on_startup_hook` takes arguments.
-    - Py_RL_STARTUP_HOOK_TAKES_ARGS
   includes-deps:
     - libedit/include
     - libedit/include/ncursesw
diff --git a/cpython-unix/patch-readline-libedit-completer-delims.patch b/cpython-unix/patch-readline-libedit-completer-delims.patch
@@ -0,0 +1,72 @@
+diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py
+index 835280f2281..6c2726d3209 100644
+--- a/Lib/test/test_readline.py
++++ b/Lib/test/test_readline.py
+@@ -5,6 +5,7 @@
+ import os
+ import sys
+ import tempfile
++import textwrap
+ import unittest
+ from test.support import verbose
+ from test.support.import_helper import import_module
+@@ -163,6 +164,25 @@ def test_auto_history_disabled(self):
+         # end, so don't expect it in the output.
+         self.assertIn(b"History length: 0", output)
+ 
++    def test_set_complete_delims(self):
++        script = textwrap.dedent("""
++            import readline
++            def complete(text, state):
++                if state == 0 and text == "$":
++                    return "$complete"
++                return None
++            if "libedit" in getattr(readline, "__doc__", ""):
++                readline.parse_and_bind(r'bind "\\t" rl_complete')
++            else:
++                readline.parse_and_bind(r'"\\t": complete')
++            readline.set_completer_delims(" \\t\\n")
++            readline.set_completer(complete)
++            print(input())
++        """)
++
++        output = run_pty(script, input=b"$\t\n")
++        self.assertIn(b"$complete", output)
++
+     def test_nonascii(self):
+         loc = locale.setlocale(locale.LC_CTYPE, None)
+         if loc in ('C', 'POSIX'):
+diff --git a/Modules/readline.c b/Modules/readline.c
+index 8c7f526d418..1e13a0e6e06 100644
+--- a/Modules/readline.c
++++ b/Modules/readline.c
+@@ -572,6 +572,13 @@ readline_set_completer_delims(PyObject *module, PyObject *string)
+     if (break_chars) {
+         free(completer_word_break_characters);
+         completer_word_break_characters = break_chars;
++#ifdef WITH_EDITLINE
++        rl_basic_word_break_characters = break_chars;
++#else
++        if (using_libedit_emulation) {
++            rl_basic_word_break_characters = break_chars;
++        }
++#endif
+         rl_completer_word_break_characters = break_chars;
+         Py_RETURN_NONE;
+     }
+@@ -1260,6 +1267,15 @@ setup_readline(readlinestate *mod_state)
+     completer_word_break_characters =
+         strdup(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?");
+         /* All nonalphanums except '.' */
++#ifdef WITH_EDITLINE
++    // libedit uses rl_basic_word_break_characters instead of
++    // rl_completer_word_break_characters as complete delimiter
++    rl_basic_word_break_characters = completer_word_break_characters;
++#else
++    if (using_libedit_emulation) {
++        rl_basic_word_break_characters = completer_word_break_characters;
++    }
++#endif
+     rl_completer_word_break_characters = completer_word_break_characters;
+ 
+     mod_state->begidx = PyLong_FromLong(0L);
diff --git a/cpython-unix/patch-readline-libedit-completions.patch b/cpython-unix/patch-readline-libedit-completions.patch
@@ -0,0 +1,52 @@
+diff --git a/Modules/readline.c b/Modules/readline.c
+index 27b89de7279..8c7f526d418 100644
+--- a/Modules/readline.c
++++ b/Modules/readline.c
+@@ -440,7 +440,7 @@ readline_set_completion_display_matches_hook_impl(PyObject *module,
+        default completion display. */
+     rl_completion_display_matches_hook =
+         readlinestate_global->completion_display_matches_hook ?
+-#if defined(_RL_FUNCTION_TYPEDEF)
++#if defined(HAVE_RL_COMPDISP_FUNC_T)
+         (rl_compdisp_func_t *)on_completion_display_matches_hook : 0;
+ #else
+         (VFunction *)on_completion_display_matches_hook : 0;
+diff --git a/configure.ac b/configure.ac
+index e1cbb7c7fbe..629b7b76c3c 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -5918,6 +5918,20 @@ if test "$py_cv_lib_readline" = yes; then
+   AC_CHECK_LIB($LIBREADLINE, append_history,
+     AC_DEFINE(HAVE_RL_APPEND_HISTORY, 1,
+       [Define if readline supports append_history]),,$READLINE_LIBS)
++
++  # in readline as well as newer editline (April 2023)
++  AC_CHECK_TYPE([rl_compdisp_func_t],
++                [AC_DEFINE([HAVE_RL_COMPDISP_FUNC_T], [1],
++                           [Define if readline supports rl_compdisp_func_t])],
++                [],
++                [
++#include <stdio.h> /* Must be first for Gnu Readline */
++#ifdef WITH_EDITLINE
++# include <editline/readline.h>
++#else
++# include <readline/readline.h>
++#endif
++                ])
+ fi
+ 
+ # End of readline checks: restore LIBS
+diff --git a/pyconfig.h.in b/pyconfig.h.in
+index 0536047f573..94d02e14c44 100644
+--- a/pyconfig.h.in
++++ b/pyconfig.h.in
+@@ -968,6 +968,9 @@
+ /* Define if you can turn off readline's signal handling. */
+ #undef HAVE_RL_CATCH_SIGNAL
+ 
++/* Define if readline supports rl_compdisp_func_t */
++#undef HAVE_RL_COMPDISP_FUNC_T
++
+ /* Define if you have readline 2.2 */
+ #undef HAVE_RL_COMPLETION_APPEND_CHARACTER
+ 
diff --git a/cpython-unix/patch-readline-libedit.patch b/cpython-unix/patch-readline-libedit.patch
@@ -1,14 +1,32 @@
 diff --git a/Modules/readline.c b/Modules/readline.c
-index 1e74f997b0..56a36e26e6 100644
+index 1e74f997b07..0c982857283 100644
 --- a/Modules/readline.c
 +++ b/Modules/readline.c
+@@ -35,7 +35,7 @@
+ #define completion_matches(x, y) \
+     rl_completion_matches((x), ((rl_compentry_func_t *)(y)))
+ #else
+-#if defined(_RL_FUNCTION_TYPEDEF)
++#ifdef USE_LIBEDIT
+ extern char **completion_matches(char *, rl_compentry_func_t *);
+ #else
+ 
+@@ -390,7 +390,7 @@ set_completion_display_matches_hook(PyObject *self, PyObject *args)
+        default completion display. */
+     rl_completion_display_matches_hook =
+         readlinestate_global->completion_display_matches_hook ?
+-#if defined(_RL_FUNCTION_TYPEDEF)
++#ifdef USE_LIBEDIT
+         (rl_compdisp_func_t *)on_completion_display_matches_hook : 0;
+ #else
+         (VFunction *)on_completion_display_matches_hook : 0;
 @@ -511,7 +511,7 @@ set the word delimiters for completion");
-
+ 
  /* _py_free_history_entry: Utility function to free a history entry. */
-
+ 
 -#if defined(RL_READLINE_VERSION) && RL_READLINE_VERSION >= 0x0500
 +#ifndef USE_LIBEDIT
-
+ 
  /* Readline version >= 5.0 introduced a timestamp field into the history entry
     structure; this needs to be freed to avoid a memory leak.  This version of
 @@ -1055,7 +1055,7 @@ flex_complete(const char *text, int start, int end)
@@ -19,7 +37,7 @@ index 1e74f997b0..56a36e26e6 100644
 +#ifndef USE_LIBEDIT
      rl_completion_suppress_append = 0;
  #endif
-
+ 
 @@ -1241,7 +1241,7 @@ readline_until_enter_or_signal(const char *prompt, int *signal)
              PyEval_SaveThread();
              if (s < 0) {
diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py
@@ -115,12 +115,11 @@
         "sha256": "828cb275b91268b1a3ea950d5c0c5eb076c678fdf005d517411f89cc8c3bb416",
         "version": "1.0.7",
     },
-    # 20221009-3.1 fails to build on musl due to an includes issue.
     "libedit": {
-        "url": "https://thrysoee.dk/editline/libedit-20210910-3.1.tar.gz",
-        "size": 524722,
-        "sha256": "6792a6a992050762edcca28ff3318cdb7de37dccf7bc30db59fcd7017eed13c5",
-        "version": "20210910-3.1",
+        "url": "https://thrysoee.dk/editline/libedit-20240808-3.1.tar.gz",
+        "size": 538611,
+        "sha256": "5f0573349d77c4a48967191cdd6634dd7aa5f6398c6a57fe037cc02696d6099f",
+        "version": "20240808-3.1",
         "library_names": ["edit"],
         "licenses": ["BSD-3-Clause"],
         "license_file": "LICENSE.libedit.txt",
