Merge pull request #1199 from google/google_sync

Google sync

Author: rchen152

pytype/abstract/_function_base.py

@@ -70,6 +70,48 @@ def match_args(self, node, args, alias_map=None, match_all_views=False):
         name = self._get_cell_variable_name(a)
         assert name is not None, "Closure variable lookup failed."
         raise function.UndefinedParameterError(name)
+    # The implementation of match_args is currently rather convoluted because we
+    # have two different implementations:
+    # * Old implementation: `_match_views` matches 'args' against 'self' one
+    #   view at a time, where a view is a mapping of every variable in args to a
+    #   particular binding. This handles generics but scales poorly with the
+    #   number of bindings per variable.
+    # * New implementation: `_match_args_sequentially` matches 'args' one at a
+    #   time. This scales better but cannot yet handle generics.
+    # Subclasses should implement the following:
+    # * _match_view(node, args, view, alias_map): this will be called repeatedly
+    #   by _match_views.
+    # * _match_args_sequentially(node, args, alias_map, match_all_views): A
+    #   sequential matching implementation.
+    # TODO(b/228241343): Get rid of _match_views and simplify match_args once
+    # _match_args_sequentially can handle generics.
+    if self._is_generic_call(args):
+      return self._match_views(node, args, alias_map, match_all_views)
+    return self._match_args_sequentially(node, args, alias_map, match_all_views)
+
+  def _is_generic_call(self, args):
+    for sig in function.get_signatures(self):
+      for t in sig.annotations.values():
+        stack = [t]
+        seen = set()
+        while stack:
+          cur = stack.pop()
+          if cur in seen:
+            continue
+          seen.add(cur)
+          if cur.formal or cur.template:
+            return True
+          if _isinstance(cur, "Union"):
+            stack.extend(cur.options)
+    if self.is_attribute_of_class and args.posargs:
+      for self_val in args.posargs[0].data:
+        for cls in self_val.cls.mro:
+          if cls.template:
+            return True
+    return False
+
+  def _match_views(self, node, args, alias_map, match_all_views):
+    """Matches all views of the given args against this function."""
     error = None
     matched = []
     arg_variables = args.get_variables()
@@ -107,6 +149,9 @@ def match_args(self, node, args, alias_map=None, match_all_views=False):
   def _match_view(self, node, args, view, alias_map):
     raise NotImplementedError(self.__class__.__name__)
 
+  def _match_args_sequentially(self, node, args, alias_map, match_all_views):
+    raise NotImplementedError(self.__class__.__name__)
+
   def __repr__(self):
     return self.full_name + "(...)"
 

pytype/abstract/_instance_base.py

@@ -9,6 +9,7 @@
 from pytype.abstract import function
 
 log = logging.getLogger(__name__)
+_isinstance = abstract_utils._isinstance  # pylint: disable=protected-access
 
 
 class SimpleValue(_base.BaseValue):
@@ -99,15 +100,18 @@ def merge_instance_type_parameter(self, node, name, value):
     else:
       self.instance_type_parameters[name] = value
 
-  def call(self, node, func, args, alias_map=None):
-    binding = func if self == func.data else self.to_binding(node)
+  def _call_helper(self, node, obj, binding, args):
+    obj_binding = binding if obj == binding.data else obj.to_binding(node)
     node, var = self.ctx.attribute_handler.get_attribute(
-        node, self, "__call__", binding)
+        node, obj, "__call__", obj_binding)
     if var is not None and var.bindings:
       return function.call_function(self.ctx, node, var, args)
     else:
       raise function.NotCallable(self)
 
+  def call(self, node, func, args, alias_map=None):
+    return self._call_helper(node, self, func, args)
+
   def argcount(self, node):
     node, var = self.ctx.attribute_handler.get_attribute(
         node, self, "__call__", self.to_binding(node))

pytype/abstract/_interpreter_function.py

@@ -234,6 +234,30 @@ def _match_view(self, node, args, view, alias_map=None):
           self.signature, args, self.ctx, bad_param=bad_arg)
     return subst
 
+  def _match_args_sequentially(self, node, args, alias_map, match_all_views):
+    def match_succeeded(match_result):
+      bad_matches, any_match = match_result
+      if not bad_matches:
+        return True
+      if match_all_views or self.ctx.options.strict_parameter_checks:
+        return False
+      return any_match
+
+    for name, arg, formal in self.signature.iter_args(args):
+      if formal is None:
+        continue
+      if name in (self.signature.varargs_name, self.signature.kwargs_name):
+        # The annotation is Tuple or Dict, but the passed arg only has to be
+        # Iterable or Mapping.
+        formal = self.ctx.convert.widen_type(formal)
+      match_result = self.ctx.matcher(node).bad_matches(arg, formal)
+      if not match_succeeded(match_result):
+        bad_arg = function.BadParam(
+            name=name, expected=formal, error_details=match_result[0][0][1])
+        raise function.WrongArgTypes(
+            self.signature, args, self.ctx, bad_param=bad_arg)
+    return [{}]
+
   def get_first_opcode(self):
     return None
 

pytype/abstract/_pytd_function.py

@@ -396,6 +396,9 @@ def _yield_matching_signatures(self, node, args, view, alias_map):
     if not matched:
       raise error  # pylint: disable=raising-bad-type
 
+  def _match_args_sequentially(self, node, args, alias_map, match_all_views):
+    return self._match_views(node, args, alias_map, match_all_views)
+
   def set_function_defaults(self, unused_node, defaults_var):
     """Attempts to set default arguments for a function's signatures.
 

pytype/abstract/_typing.py

@@ -75,6 +75,9 @@ def __init__(self, name, ctx, base_cls):
     super().__init__(name, ctx)
     self.base_cls = base_cls
 
+  def __repr__(self):
+    return "AnnotationContainer(%s)" % self.name
+
   def _sub_annotation(
       self, annot: _base.BaseValue, subst: Mapping[str, _base.BaseValue]
   ) -> _base.BaseValue:
@@ -277,7 +280,7 @@ def _build_value(self, node, inner, ellipses):
                 root_node, container=abstract_utils.DUMMY_CONTAINER)
         else:
           actual = param_value.instantiate(root_node)
-        bad = self.ctx.matcher(root_node).bad_matches(actual, formal_param)
+        bad, _ = self.ctx.matcher(root_node).bad_matches(actual, formal_param)
         if bad:
           if not isinstance(param_value, TypeParameter):
             # If param_value is not a TypeVar, we substitute in TypeVar bounds
@@ -300,6 +303,9 @@ def _build_value(self, node, inner, ellipses):
       self.ctx.errorlog.invalid_annotation(self.ctx.vm.frames, e.annot, e.error)
       return self.ctx.convert.unsolvable
 
+  def call(self, node, func, args, alias_map=None):
+    return self._call_helper(node, self.base_cls, func, args)
+
 
 class TypeParameter(_base.BaseValue):
   """Parameter of a type."""

pytype/abstract/function.py

@@ -30,7 +30,7 @@ def get_signatures(func):
     return [sig.drop_first_parameter() for sig in sigs]  # drop "self"
   elif _isinstance(func, ("ClassMethod", "StaticMethod")):
     return get_signatures(func.method)
-  elif _isinstance(func, "SimpleFunction"):
+  elif _isinstance(func, "SignedFunction"):
     return [func.signature]
   elif _isinstance(func.cls, "CallableClass"):
     return [Signature.from_callable(func.cls)]

pytype/context.py

@@ -164,7 +164,7 @@ def check_annotation_type_mismatch(
         typ, ("typing.ClassVar", "dataclasses.InitVar"))
     if contained_type:
       typ = contained_type
-    bad = self.matcher(node).bad_matches(value, typ)
+    bad, _ = self.matcher(node).bad_matches(value, typ)
     for view, error_details in bad:
       binding = view[value]
       self.errorlog.annotation_type_mismatch(

pytype/load_pytd.py

@@ -286,10 +286,9 @@ def log_module_not_found(self, module_name):
 class _Resolver:
   """Resolve symbols in a pytd tree."""
 
-  def __init__(self, builtins_ast, enable_nested_classes):
+  def __init__(self, builtins_ast):
     self.builtins_ast = builtins_ast
     self.allow_singletons = False
-    self._enable_nested_classes = enable_nested_classes
 
   def _lookup(self, visitor, mod_ast, lookup_ast):
     if lookup_ast:
@@ -298,9 +297,7 @@ def _lookup(self, visitor, mod_ast, lookup_ast):
     return mod_ast
 
   def resolve_local_types(self, mod_ast, *, lookup_ast=None):
-    local_lookup = visitors.LookupLocalTypes(
-        self.allow_singletons,
-        enable_nested_classes=self._enable_nested_classes)
+    local_lookup = visitors.LookupLocalTypes(self.allow_singletons)
     return self._lookup(local_lookup, mod_ast, lookup_ast)
 
   def resolve_builtin_types(self, mod_ast, *, lookup_ast=None):
@@ -376,7 +373,7 @@ def __init__(self, options, modules=None):
     self._path_finder = _PathFinder(options)
     self._builtin_loader = builtin_stubs.BuiltinLoader(
         parser.PyiOptions.from_toplevel_options(options))
-    self._resolver = _Resolver(self.builtins, options.enable_nested_classes)
+    self._resolver = _Resolver(self.builtins)
     self._import_name_cache = {}  # performance cache
     self._aliases = {}
     self._prefixes = set()

pytype/matcher.py

@@ -175,16 +175,21 @@ def bad_matches(self, var, other_type):
       var: A cfg.Variable, containing instances.
       other_type: An instance of BaseValue.
     Returns:
-      A list of all the views of var that didn't match.
+      A pair of:
+      * A list of all the views of var that didn't match.
+      * Whether at least one view matched.
+      TODO(b/63407497): We should be able to get rid of the second value once we
+      start requiring that all views match.
     """
     bad = []
     if (var.data == [self.ctx.convert.unsolvable] or
         other_type == self.ctx.convert.unsolvable):
       # An unsolvable matches everything. Since bad_matches doesn't need to
       # compute substitutions, we can return immediately.
-      return bad
+      return bad, True
     views = abstract_utils.get_views([var], self._node)
     skip_future = None
+    any_match = False
     while True:
       try:
         view = views.send(skip_future)
@@ -200,7 +205,8 @@ def bad_matches(self, var, other_type):
         skip_future = False
       else:
         skip_future = True
-    return bad
+        any_match = True
+    return bad, any_match
 
   def match_from_mro(self, left, other_type, allow_compat_builtins=True):
     """Checks a type's MRO for a match for a formal type.
@@ -989,7 +995,7 @@ def _match_dict_against_typed_dict(self, left, other_type):
       if k not in fields:
         continue
       typ = abstract_utils.get_atomic_value(fields[k])
-      b = self.bad_matches(v, typ)
+      b, _ = self.bad_matches(v, typ)
       if b:
         bad.append((k, v, typ, b))
     if missing or extra or bad:

pytype/matcher_test.py

@@ -508,14 +508,14 @@ def test_bad_matches(self):
         self.matcher.bad_matches(
             abstract.TypeParameter("T",
                                    self.ctx).to_variable(self.ctx.root_node),
-            self.ctx.convert.unsolvable))
+            self.ctx.convert.unsolvable)[0])
 
   def test_bad_matches_no_match(self):
     self.assertTrue(
         self.matcher.bad_matches(
             abstract.TypeParameter("T",
                                    self.ctx).to_variable(self.ctx.root_node),
-            self.ctx.convert.int_type))
+            self.ctx.convert.int_type)[0])
 
   def test_any(self):
     self.assertMatch(