Fix completion tuple (ipython#14594)

In progress work toward ipython#14585


guarded eval strip leading characters until it find soemthing, this is
problematic as `(1, x`, becomes valid after 1 char strip: `1, x` is a
tuple;

So now we trim until it is valid an not a tuple. 

This is still imperfect as things like `(1, a[" "].y` will be trimmed to
`y`, while it should stop with `a[" "].y` ?

I think maybe we should back-propagate; build back up from `y`, to `a["
"].y`, greedily until we get the last valid expression – skipping any
unbalanced parentheses/quotes if we encounter imblanced.

Author: Carreau

IPython/core/completer.py

@@ -184,6 +184,7 @@
 import inspect
 import itertools
 import keyword
+import ast
 import os
 import re
 import string
@@ -347,7 +348,7 @@ def provisionalcompleter(action='ignore'):
         yield
 
 
-def has_open_quotes(s):
+def has_open_quotes(s: str) -> Union[str, bool]:
     """Return whether a string has open quotes.
 
     This simply counts whether the number of quote characters of either type in
@@ -368,7 +369,7 @@ def has_open_quotes(s):
         return False
 
 
-def protect_filename(s, protectables=PROTECTABLES):
+def protect_filename(s: str, protectables: str = PROTECTABLES) -> str:
     """Escape a string to protect certain characters."""
     if set(s) & set(protectables):
         if sys.platform == "win32":
@@ -449,11 +450,11 @@ def completions_sorting_key(word):
 
     if word.startswith('%%'):
         # If there's another % in there, this is something else, so leave it alone
-        if not "%" in word[2:]:
+        if "%" not in word[2:]:
             word = word[2:]
             prio2 = 2
     elif word.startswith('%'):
-        if not "%" in word[1:]:
+        if "%" not in word[1:]:
             word = word[1:]
             prio2 = 1
 
@@ -752,7 +753,7 @@ def completion_matcher(
     priority: Optional[float] = None,
     identifier: Optional[str] = None,
     api_version: int = 1,
-):
+) -> Callable[[Matcher], Matcher]:
     """Adds attributes describing the matcher.
 
     Parameters
@@ -961,8 +962,8 @@ def delims(self, delims):
     def split_line(self, line, cursor_pos=None):
         """Split a line of text with a cursor at the given position.
         """
-        l = line if cursor_pos is None else line[:cursor_pos]
-        return self._delim_re.split(l)[-1]
+        cut_line = line if cursor_pos is None else line[:cursor_pos]
+        return self._delim_re.split(cut_line)[-1]
 
 
 
@@ -1141,8 +1142,13 @@ def attr_matches(self, text):
         """
         return self._attr_matches(text)[0]
 
-    def _attr_matches(self, text, include_prefix=True) -> Tuple[Sequence[str], str]:
-        m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer)
+    # we simple attribute matching with normal identifiers.
+    _ATTR_MATCH_RE = re.compile(r"(.+)\.(\w*)$")
+
+    def _attr_matches(
+        self, text: str, include_prefix: bool = True
+    ) -> Tuple[Sequence[str], str]:
+        m2 = self._ATTR_MATCH_RE.match(self.line_buffer)
         if not m2:
             return [], ""
         expr, attr = m2.group(1, 2)
@@ -1204,6 +1210,30 @@ def _attr_matches(self, text, include_prefix=True) -> Tuple[Sequence[str], str]:
             "." + attr,
         )
 
+    def _trim_expr(self, code: str) -> str:
+        """
+        Trim the code until it is a valid expression and not a tuple;
+
+        return the trimmed expression for guarded_eval.
+        """
+        while code:
+            code = code[1:]
+            try:
+                res = ast.parse(code)
+            except SyntaxError:
+                continue
+
+            assert res is not None
+            if len(res.body) != 1:
+                continue
+            expr = res.body[0].value
+            if isinstance(expr, ast.Tuple) and not code[-1] == ")":
+                # we skip implicit tuple, like when trimming `fun(a,b`<completion>
+                # as `a,b` would be a tuple, and we actually expect to get only `b`
+                continue
+            return code
+        return ""
+
     def _evaluate_expr(self, expr):
         obj = not_found
         done = False
@@ -1225,14 +1255,14 @@ def _evaluate_expr(self, expr):
                 # e.g. user starts `(d[`, so we get `expr = '(d'`,
                 # where parenthesis is not closed.
                 # TODO: make this faster by reusing parts of the computation?
-                expr = expr[1:]
+                expr = self._trim_expr(expr)
         return obj
 
 def get__all__entries(obj):
     """returns the strings in the __all__ attribute"""
     try:
         words = getattr(obj, '__all__')
-    except:
+    except Exception:
         return []
 
     return [w for w in words if isinstance(w, str)]
@@ -1447,7 +1477,7 @@ def filter_prefix_tuple(key):
         try:
             if not str_key.startswith(prefix_str):
                 continue
-        except (AttributeError, TypeError, UnicodeError) as e:
+        except (AttributeError, TypeError, UnicodeError):
             # Python 3+ TypeError on b'a'.startswith('a') or vice-versa
             continue
 
@@ -1495,7 +1525,7 @@ def cursor_to_position(text:str, line:int, column:int)->int:
     lines = text.split('\n')
     assert line <= len(lines), '{} <= {}'.format(str(line), str(len(lines)))
 
-    return sum(len(l) + 1 for l in lines[:line]) + column
+    return sum(len(line) + 1 for line in lines[:line]) + column
 
 def position_to_cursor(text:str, offset:int)->Tuple[int, int]:
     """
@@ -2112,7 +2142,7 @@ def magic_matcher(self, context: CompletionContext) -> SimpleMatcherResult:
         result["suppress"] = is_magic_prefix and bool(result["completions"])
         return result
 
-    def magic_matches(self, text: str):
+    def magic_matches(self, text: str) -> List[str]:
         """Match magics.
 
         .. deprecated:: 8.6
@@ -2469,7 +2499,8 @@ def python_func_kw_matches(self, text):
         # parenthesis before the cursor
         # e.g. for "foo (1+bar(x), pa<cursor>,a=1)", the candidate is "foo"
         tokens = regexp.findall(self.text_until_cursor)
-        iterTokens = reversed(tokens); openPar = 0
+        iterTokens = reversed(tokens)
+        openPar = 0
 
         for token in iterTokens:
             if token == ')':
@@ -2489,7 +2520,8 @@ def python_func_kw_matches(self, text):
             try:
                 ids.append(next(iterTokens))
                 if not isId(ids[-1]):
-                    ids.pop(); break
+                    ids.pop()
+                    break
                 if not next(iterTokens) == '.':
                     break
             except StopIteration:
@@ -3215,7 +3247,7 @@ def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None,
                 else:
                     api_version = _get_matcher_api_version(matcher)
                     raise ValueError(f"Unsupported API version {api_version}")
-            except:
+            except BaseException:
                 # Show the ugly traceback if the matcher causes an
                 # exception, but do NOT crash the kernel!
                 sys.excepthook(*sys.exc_info())

IPython/core/tests/test_completer.py

@@ -9,10 +9,10 @@
 import sys
 import textwrap
 import unittest
+import random
 
 from importlib.metadata import version
 
-
 from contextlib import contextmanager
 
 from traitlets.config.loader import Config
@@ -21,6 +21,7 @@
 from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory
 from IPython.utils.generics import complete_object
 from IPython.testing import decorators as dec
+from IPython.core.latex_symbols import latex_symbols
 
 from IPython.core.completer import (
     Completion,
@@ -31,11 +32,24 @@
     completion_matcher,
     SimpleCompletion,
     CompletionContext,
+    _unicode_name_compute,
+    _UNICODE_RANGES,
 )
 
 from packaging.version import parse
 
 
+@contextmanager
+def jedi_status(status: bool):
+    completer = get_ipython().Completer
+    try:
+        old = completer.use_jedi
+        completer.use_jedi = status
+        yield
+    finally:
+        completer.use_jedi = old
+
+
 # -----------------------------------------------------------------------------
 # Test functions
 # -----------------------------------------------------------------------------
@@ -66,7 +80,7 @@ def ranges(i):
     rg = list(ranges(valid))
     lens = []
     gap_lens = []
-    pstart, pstop = 0, 0
+    _pstart, pstop = 0, 0
     for start, stop in rg:
         lens.append(stop - start)
         gap_lens.append(
@@ -77,7 +91,7 @@ def ranges(i):
                 f"{round((start - pstop)/0xe01f0*100)}%",
             )
         )
-        pstart, pstop = start, stop
+        _pstart, pstop = start, stop
 
     return sorted(gap_lens)[-1]
 
@@ -87,7 +101,6 @@ def test_unicode_range():
     Test that the ranges we test for unicode names give the same number of
     results than testing the full length.
     """
-    from IPython.core.completer import _unicode_name_compute, _UNICODE_RANGES
 
     expected_list = _unicode_name_compute([(0, 0x110000)])
     test = _unicode_name_compute(_UNICODE_RANGES)
@@ -148,45 +161,45 @@ def custom_matchers(matchers):
         ip.Completer.custom_matchers.clear()
 
 
-def test_protect_filename():
-    if sys.platform == "win32":
-        pairs = [
-            ("abc", "abc"),
-            (" abc", '" abc"'),
-            ("a bc", '"a bc"'),
-            ("a  bc", '"a  bc"'),
-            ("  bc", '"  bc"'),
-        ]
-    else:
-        pairs = [
-            ("abc", "abc"),
-            (" abc", r"\ abc"),
-            ("a bc", r"a\ bc"),
-            ("a  bc", r"a\ \ bc"),
-            ("  bc", r"\ \ bc"),
-            # On posix, we also protect parens and other special characters.
-            ("a(bc", r"a\(bc"),
-            ("a)bc", r"a\)bc"),
-            ("a( )bc", r"a\(\ \)bc"),
-            ("a[1]bc", r"a\[1\]bc"),
-            ("a{1}bc", r"a\{1\}bc"),
-            ("a#bc", r"a\#bc"),
-            ("a?bc", r"a\?bc"),
-            ("a=bc", r"a\=bc"),
-            ("a\\bc", r"a\\bc"),
-            ("a|bc", r"a\|bc"),
-            ("a;bc", r"a\;bc"),
-            ("a:bc", r"a\:bc"),
-            ("a'bc", r"a\'bc"),
-            ("a*bc", r"a\*bc"),
-            ('a"bc', r"a\"bc"),
-            ("a^bc", r"a\^bc"),
-            ("a&bc", r"a\&bc"),
-        ]
-    # run the actual tests
-    for s1, s2 in pairs:
-        s1p = completer.protect_filename(s1)
-        assert s1p == s2
+if sys.platform == "win32":
+    pairs = [
+        ("abc", "abc"),
+        (" abc", '" abc"'),
+        ("a bc", '"a bc"'),
+        ("a  bc", '"a  bc"'),
+        ("  bc", '"  bc"'),
+    ]
+else:
+    pairs = [
+        ("abc", "abc"),
+        (" abc", r"\ abc"),
+        ("a bc", r"a\ bc"),
+        ("a  bc", r"a\ \ bc"),
+        ("  bc", r"\ \ bc"),
+        # On posix, we also protect parens and other special characters.
+        ("a(bc", r"a\(bc"),
+        ("a)bc", r"a\)bc"),
+        ("a( )bc", r"a\(\ \)bc"),
+        ("a[1]bc", r"a\[1\]bc"),
+        ("a{1}bc", r"a\{1\}bc"),
+        ("a#bc", r"a\#bc"),
+        ("a?bc", r"a\?bc"),
+        ("a=bc", r"a\=bc"),
+        ("a\\bc", r"a\\bc"),
+        ("a|bc", r"a\|bc"),
+        ("a;bc", r"a\;bc"),
+        ("a:bc", r"a\:bc"),
+        ("a'bc", r"a\'bc"),
+        ("a*bc", r"a\*bc"),
+        ('a"bc', r"a\"bc"),
+        ("a^bc", r"a\^bc"),
+        ("a&bc", r"a\&bc"),
+    ]
+
+
+@pytest.mark.parametrize("s1,expected", pairs)
+def test_protect_filename(s1, expected):
+    assert completer.protect_filename(s1) == expected
 
 
 def check_line_split(splitter, test_specs):
@@ -297,8 +310,6 @@ def test_unicode_completions(self):
             self.assertIsInstance(matches, list)
 
     def test_latex_completions(self):
-        from IPython.core.latex_symbols import latex_symbols
-        import random
 
         ip = get_ipython()
         # Test some random unicode symbols
@@ -1734,6 +1745,45 @@ def _(expected):
             _(["completion_a"])
 
 
+@pytest.mark.parametrize(
+    "setup,code,expected,not_expected",
+    [
+        ('a="str"; b=1', "(a, b.", [".bit_count", ".conjugate"], [".count"]),
+        ('a="str"; b=1', "(a, b).", [".count"], [".bit_count", ".capitalize"]),
+        ('x="str"; y=1', "x = {1, y.", [".bit_count"], [".count"]),
+        ('x="str"; y=1', "x = [1, y.", [".bit_count"], [".count"]),
+        ('x="str"; y=1; fun=lambda x:x', "x = fun(1, y.", [".bit_count"], [".count"]),
+    ],
+)
+def test_misc_no_jedi_completions(setup, code, expected, not_expected):
+    ip = get_ipython()
+    c = ip.Completer
+    ip.ex(setup)
+    with provisionalcompleter(), jedi_status(False):
+        matches = c.all_completions(code)
+        assert set(expected) - set(matches) == set(), set(matches)
+        assert set(matches).intersection(set(not_expected)) == set()
+
+
+@pytest.mark.parametrize(
+    "code,expected",
+    [
+        (" (a, b", "b"),
+        ("(a, b", "b"),
+        ("(a, b)", ""),  # trim always start by trimming
+        (" (a, b)", "(a, b)"),
+        (" [a, b]", "[a, b]"),
+        (" a, b", "b"),
+        ("x = {1, y", "y"),
+        ("x = [1, y", "y"),
+        ("x = fun(1, y", "y"),
+    ],
+)
+def test_trim_expr(code, expected):
+    c = get_ipython().Completer
+    assert c._trim_expr(code) == expected
+
+
 @pytest.mark.parametrize(
     "input, expected",
     [