Merge branch 'python:master' into callable_pretty

MechanicalConstruct · web-flow · commit 0fa975d3b40d · 2024-12-04T12:27:00.000-05:00
diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst
@@ -166,6 +166,10 @@ imports.
 
     For more details, see :ref:`ignore-missing-imports`.
 
+.. option:: --follow-untyped-imports
+
+    This flag makes mypy analyze imports without stubs or a py.typed marker.
+
 .. option:: --follow-imports {normal,silent,skip,error}
 
     This flag adjusts how mypy follows imported modules that were not
@@ -537,12 +541,12 @@ potentially problematic or redundant in some way.
 
         This limitation will be removed in future releases of mypy.
 
-.. option:: --report-deprecated-as-error
+.. option:: --report-deprecated-as-note
 
-    By default, mypy emits notes if your code imports or uses deprecated
-    features.    This flag converts such notes to errors, causing mypy to
-    eventually finish with a non-zero exit code.  Features are considered
-    deprecated when decorated with ``warnings.deprecated``.
+    If error code ``deprecated`` is enabled, mypy emits errors if your code
+    imports or uses deprecated features. This flag converts such errors to
+    notes, causing mypy to eventually finish with a zero exit code. Features
+    are considered deprecated when decorated with ``warnings.deprecated``.
 
 .. _miscellaneous-strictness-flags:
 
diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst
@@ -315,6 +315,18 @@ section of the command line docs.
     match the name of the *imported* module, not the module containing the
     import statement.
 
+.. confval:: follow_untyped_imports
+
+    :type: boolean
+    :default: False
+
+    Typechecks imports from modules that do not have stubs or a py.typed marker.
+
+    If this option is used in a per-module section, the module name should
+    match the name of the *imported* module, not the module containing the
+    import statement. Note that scanning all unannotated modules might
+    significantly increase the runtime of your mypy calls.
+
 .. confval:: follow_imports
 
     :type: string
diff --git a/docs/source/error_code_list2.rst b/docs/source/error_code_list2.rst
@@ -236,13 +236,13 @@ incorrect control flow or conditional checks that are accidentally always true o
 Check that imported or used feature is deprecated [deprecated]
 --------------------------------------------------------------
 
-By default, mypy generates a note if your code imports a deprecated feature explicitly with a
+If you use :option:`--enable-error-code deprecated <mypy --enable-error-code>`,
+mypy generates an error if your code imports a deprecated feature explicitly with a
 ``from mod import depr`` statement or uses a deprecated feature imported otherwise or defined
 locally.  Features are considered deprecated when decorated with ``warnings.deprecated``, as
-specified in `PEP 702 <https://peps.python.org/pep-0702>`_.  You can silence single notes via
-``# type: ignore[deprecated]`` or turn off this check completely via ``--disable-error-code=deprecated``.
-Use the :option:`--report-deprecated-as-error <mypy --report-deprecated-as-error>` option for
-more strictness, which turns all such notes into errors.
+specified in `PEP 702 <https://peps.python.org/pep-0702>`_.
+Use the :option:`--report-deprecated-as-note <mypy --report-deprecated-as-note>` option to
+turn all such errors into notes.
 
 .. note::
 
diff --git a/docs/source/running_mypy.rst b/docs/source/running_mypy.rst
@@ -321,6 +321,12 @@ not catch errors in its use.
     recommend avoiding ``--ignore-missing-imports`` if possible: it's equivalent
     to adding a ``# type: ignore`` to all unresolved imports in your codebase.
 
+4.  To make mypy typecheck imports from modules without stubs or a py.typed
+    marker, you can set the :option:`--follow-untyped-imports <mypy --follow-untyped-imports>`
+    command line flag or set the :confval:`follow_untyped_imports` config file option to True,
+    either in the global section of your mypy config file, or individually on a
+    per-module basis.
+
 
 Library stubs not installed
 ---------------------------
diff --git a/mypy/checker.py b/mypy/checker.py
@@ -7696,7 +7696,7 @@ def warn_deprecated(self, node: SymbolNode | None, context: Context) -> None:
             and ((deprecated := node.deprecated) is not None)
             and not self.is_typeshed_stub
         ):
-            warn = self.msg.fail if self.options.report_deprecated_as_error else self.msg.note
+            warn = self.msg.note if self.options.report_deprecated_as_note else self.msg.fail
             warn(deprecated, context, code=codes.DEPRECATED)
 
 
diff --git a/mypy/errorcodes.py b/mypy/errorcodes.py
@@ -308,6 +308,7 @@ def __hash__(self) -> int:
     "deprecated",
     "Warn when importing or using deprecated (overloaded) functions, methods or classes",
     "General",
+    default_enabled=False,
 )
 
 # This copy will not include any error codes defined later in the plugins.
diff --git a/mypy/main.py b/mypy/main.py
@@ -580,6 +580,11 @@ def add_invertible_flag(
         action="store_true",
         help="Silently ignore imports of missing modules",
     )
+    imports_group.add_argument(
+        "--follow-untyped-imports",
+        action="store_true",
+        help="Typecheck modules without stubs or py.typed marker",
+    )
     imports_group.add_argument(
         "--follow-imports",
         choices=["normal", "silent", "skip", "error"],
@@ -805,10 +810,10 @@ def add_invertible_flag(
         group=lint_group,
     )
     add_invertible_flag(
-        "--report-deprecated-as-error",
+        "--report-deprecated-as-note",
         default=False,
         strict_flag=False,
-        help="Report importing or using deprecated features as errors instead of notes",
+        help="Report importing or using deprecated features as notes instead of errors",
         group=lint_group,
     )
 
diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py
@@ -334,10 +334,11 @@ def _typeshed_has_version(self, module: str) -> bool:
         return version >= min_version and (max_version is None or version <= max_version)
 
     def _find_module_non_stub_helper(
-        self, components: list[str], pkg_dir: str
+        self, id: str, pkg_dir: str
     ) -> OnePackageDir | ModuleNotFoundReason:
         plausible_match = False
         dir_path = pkg_dir
+        components = id.split(".")
         for index, component in enumerate(components):
             dir_path = os_path_join(dir_path, component)
             if self.fscache.isfile(os_path_join(dir_path, "py.typed")):
@@ -350,6 +351,10 @@ def _find_module_non_stub_helper(
             if not self.fscache.isdir(dir_path):
                 break
         if plausible_match:
+            if self.options:
+                module_specific_options = self.options.clone_for_module(id)
+                if module_specific_options.follow_untyped_imports:
+                    return os.path.join(pkg_dir, *components[:-1]), False
             return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS
         else:
             return ModuleNotFoundReason.NOT_FOUND
@@ -463,7 +468,7 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult:
                             third_party_stubs_dirs.append((path, True))
                     else:
                         third_party_stubs_dirs.append((path, True))
-            non_stub_match = self._find_module_non_stub_helper(components, pkg_dir)
+            non_stub_match = self._find_module_non_stub_helper(id, pkg_dir)
             if isinstance(non_stub_match, ModuleNotFoundReason):
                 if non_stub_match is ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS:
                     found_possible_third_party_missing_type_hints = True
diff --git a/mypy/options.py b/mypy/options.py
@@ -42,6 +42,7 @@ class BuildType:
     "extra_checks",
     "follow_imports_for_stubs",
     "follow_imports",
+    "follow_untyped_imports",
     "ignore_errors",
     "ignore_missing_imports",
     "implicit_optional",
@@ -113,6 +114,8 @@ def __init__(self) -> None:
         self.ignore_missing_imports = False
         # Is ignore_missing_imports set in a per-module section
         self.ignore_missing_imports_per_module = False
+        # Typecheck modules without stubs or py.typed marker
+        self.follow_untyped_imports = False
         self.follow_imports = "normal"  # normal|silent|skip|error
         # Whether to respect the follow_imports setting even for stub files.
         # Intended to be used for disabling specific stubs.
@@ -174,7 +177,7 @@ def __init__(self) -> None:
         self.warn_return_any = False
 
         # Report importing or using deprecated features as errors instead of notes.
-        self.report_deprecated_as_error = False
+        self.report_deprecated_as_note = False
 
         # Warn about unused '# type: ignore' comments
         self.warn_unused_ignores = False
diff --git a/mypy/typeanal.py b/mypy/typeanal.py
@@ -790,7 +790,7 @@ def check_and_warn_deprecated(self, info: TypeInfo, ctx: Context) -> None:
                 if isinstance(imp, ImportFrom) and any(info.name == n[0] for n in imp.names):
                     break
             else:
-                warn = self.fail if self.options.report_deprecated_as_error else self.note
+                warn = self.note if self.options.report_deprecated_as_note else self.fail
                 warn(deprecated, ctx, code=codes.DEPRECATED)
 
     def analyze_type_with_type_info(
diff --git a/mypyc/doc/bytes_operations.rst b/mypyc/doc/bytes_operations.rst
@@ -0,0 +1,46 @@
+.. _bytes-ops:
+
+Native bytes operations
+========================
+
+These ``bytes`` operations have fast, optimized implementations. Other
+bytes operations use generic implementations that are often slower.
+
+Construction
+------------
+
+* Bytes literal
+* ``bytes(x: list)``
+
+Operators
+---------
+
+* Concatenation (``b1 + b2``)
+* Indexing (``b[n]``)
+* Slicing (``b[n:m]``, ``b[n:]``, ``b[:m]``)
+* Comparisons (``==``, ``!=``)
+
+.. _bytes-methods:
+
+Methods
+-------
+
+* ``b.decode()``
+* ``b.decode(encoding: str)``
+* ``b.decode(encoding: str, errors: str)``
+* ``b.join(x: Iterable)``
+
+.. note::
+
+    :ref:`str.encode() <str-methods>` is also optimized.
+
+Formatting
+----------
+
+A subset of % formatting operations are optimized (``b"..." % (...)``).
+
+Functions
+---------
+
+* ``len(b: bytes)``
+* ``ord(b: bytes)``
diff --git a/mypyc/doc/index.rst b/mypyc/doc/index.rst
@@ -36,6 +36,7 @@ generate fast code.
    bool_operations
    float_operations
    str_operations
+   bytes_operations
    list_operations
    dict_operations
    set_operations
diff --git a/mypyc/doc/str_operations.rst b/mypyc/doc/str_operations.rst
@@ -22,9 +22,14 @@ Operators
 * Comparisons (``==``, ``!=``)
 * Augmented assignment (``s1 += s2``)
 
+.. _str-methods:
+
 Methods
 -------
 
+* ``s.encode()``
+* ``s.encode(encoding: str)``
+* ``s.encode(encoding: str, errors: str)``
 * ``s1.endswith(s2: str)``
 * ``s.join(x: Iterable)``
 * ``s.replace(old: str, new: str)``
@@ -33,3 +38,22 @@ Methods
 * ``s.split(sep: str)``
 * ``s.split(sep: str, maxsplit: int)``
 * ``s1.startswith(s2: str)``
+
+.. note::
+
+    :ref:`bytes.decode() <bytes-methods>` is also optimized.
+
+Formatting
+----------
+
+A subset of these common string formatting expressions are optimized:
+
+* F-strings
+* ``"...".format(...)``
+* ``"..." % (...)``
+
+Functions
+---------
+
+* ``len(s: str)``
+* ``ord(s: str)``
diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py
@@ -19,6 +19,7 @@
 from mypy.nodes import (
     ARG_NAMED,
     ARG_POS,
+    BytesExpr,
     CallExpr,
     DictExpr,
     Expression,
@@ -877,3 +878,13 @@ def translate_float(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Valu
         # No-op float conversion.
         return builder.accept(arg)
     return None
+
+
+@specialize_function("builtins.ord")
+def translate_ord(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None:
+    if len(expr.args) != 1 or expr.arg_kinds[0] != ARG_POS:
+        return None
+    arg = expr.args[0]
+    if isinstance(arg, (StrExpr, BytesExpr)) and len(arg.value) == 1:
+        return Integer(ord(arg.value))
+    return None
diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h
@@ -730,6 +730,7 @@ bool CPyStr_IsTrue(PyObject *obj);
 Py_ssize_t CPyStr_Size_size_t(PyObject *str);
 PyObject *CPy_Decode(PyObject *obj, PyObject *encoding, PyObject *errors);
 PyObject *CPy_Encode(PyObject *obj, PyObject *encoding, PyObject *errors);
+CPyTagged CPyStr_Ord(PyObject *obj);
 
 
 // Bytes operations
@@ -740,6 +741,7 @@ PyObject *CPyBytes_GetSlice(PyObject *obj, CPyTagged start, CPyTagged end);
 CPyTagged CPyBytes_GetItem(PyObject *o, CPyTagged index);
 PyObject *CPyBytes_Concat(PyObject *a, PyObject *b);
 PyObject *CPyBytes_Join(PyObject *sep, PyObject *iter);
+CPyTagged CPyBytes_Ord(PyObject *obj);
 
 
 int CPyBytes_Compare(PyObject *left, PyObject *right);
diff --git a/mypyc/lib-rt/bytes_ops.c b/mypyc/lib-rt/bytes_ops.c
@@ -141,3 +141,20 @@ PyObject *CPyBytes_Build(Py_ssize_t len, ...) {
 
     return (PyObject *)ret;
 }
+
+
+CPyTagged CPyBytes_Ord(PyObject *obj) {
+    if (PyBytes_Check(obj)) {
+        Py_ssize_t s = PyBytes_GET_SIZE(obj);
+        if (s == 1) {
+            return (unsigned char)(PyBytes_AS_STRING(obj)[0]) << 1;
+        }
+    } else if (PyByteArray_Check(obj)) {
+        Py_ssize_t s = PyByteArray_GET_SIZE(obj);
+        if (s == 1) {
+            return (unsigned char)(PyByteArray_AS_STRING(obj)[0]) << 1;
+        }
+    }
+    PyErr_SetString(PyExc_TypeError, "ord() expects a character");
+    return CPY_INT_TAG;
+}
diff --git a/mypyc/lib-rt/str_ops.c b/mypyc/lib-rt/str_ops.c
@@ -243,3 +243,15 @@ PyObject *CPy_Encode(PyObject *obj, PyObject *encoding, PyObject *errors) {
         return NULL;
     }
 }
+
+
+CPyTagged CPyStr_Ord(PyObject *obj) {
+    Py_ssize_t s = PyUnicode_GET_LENGTH(obj);
+    if (s == 1) {
+        int kind = PyUnicode_KIND(obj);
+        return PyUnicode_READ(kind, PyUnicode_DATA(obj), 0) << 1;
+    }
+    PyErr_Format(
+        PyExc_TypeError, "ord() expected a character, but a string of length %zd found", s);
+    return CPY_INT_TAG;
+}
diff --git a/mypyc/primitives/bytes_ops.py b/mypyc/primitives/bytes_ops.py
@@ -99,3 +99,11 @@
     error_kind=ERR_MAGIC,
     var_arg_type=bytes_rprimitive,
 )
+
+function_op(
+    name="builtins.ord",
+    arg_types=[bytes_rprimitive],
+    return_type=int_rprimitive,
+    c_function_name="CPyBytes_Ord",
+    error_kind=ERR_MAGIC,
+)
diff --git a/mypyc/primitives/str_ops.py b/mypyc/primitives/str_ops.py
@@ -251,3 +251,11 @@
     c_function_name="CPy_Encode",
     error_kind=ERR_MAGIC,
 )
+
+function_op(
+    name="builtins.ord",
+    arg_types=[str_rprimitive],
+    return_type=int_rprimitive,
+    c_function_name="CPyStr_Ord",
+    error_kind=ERR_MAGIC,
+)
diff --git a/mypyc/test-data/fixtures/testutil.py b/mypyc/test-data/fixtures/testutil.py
@@ -45,7 +45,7 @@ def assertRaises(typ: type, msg: str = '') -> Iterator[None]:
     try:
         yield
     except Exception as e:
-        assert isinstance(e, typ), f"{e!r} is not a {typ.__name__}"
+        assert type(e) is typ, f"{e!r} is not a {typ.__name__}"
         assert msg in str(e), f'Message "{e}" does not match "{msg}"'
     else:
         assert False, f"Expected {typ.__name__} but got no exception"
diff --git a/mypyc/test-data/irbuild-str.test b/mypyc/test-data/irbuild-str.test
diff --git a/mypyc/test-data/run-bytes.test b/mypyc/test-data/run-bytes.test
diff --git a/mypyc/test-data/run-i64.test b/mypyc/test-data/run-i64.test
diff --git a/mypyc/test-data/run-misc.test b/mypyc/test-data/run-misc.test
diff --git a/mypyc/test-data/run-primitives.test b/mypyc/test-data/run-primitives.test
diff --git a/mypyc/test-data/run-strings.test b/mypyc/test-data/run-strings.test
diff --git a/test-data/unit/check-deprecated.test b/test-data/unit/check-deprecated.test
diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test
diff --git a/test-data/unit/pep561.test b/test-data/unit/pep561.test

Original file line number	Diff line number	Diff line change
`@@ -308,6 +308,7 @@ def __hash__(self) -> int:`
`308`	`308`	`"deprecated",`
`309`	`309`	`"Warn when importing or using deprecated (overloaded) functions, methods or classes",`
`310`	`310`	`"General",`
	`311`	`+ default_enabled=False,`
`311`	`312`	`)`
`312`	`313`
`313`	`314`	`# This copy will not include any error codes defined later in the plugins.`