Merge branch 'main' into make-resource-tracker-reentrant-safe

Author: gpshead

.github/CODEOWNERS

@@ -188,7 +188,7 @@ Include/internal/pycore_time.h  @pganssle @abalkin
 
 # AST
 Python/ast.c                  @isidentical @JelleZijlstra @eclips4
-Python/ast_opt.c              @isidentical @eclips4
+Python/ast_preprocess.c       @isidentical @eclips4
 Parser/asdl.py                @isidentical @JelleZijlstra @eclips4
 Parser/asdl_c.py              @isidentical @JelleZijlstra @eclips4
 Lib/ast.py                    @isidentical @JelleZijlstra @eclips4

Doc/library/annotationlib.rst

@@ -40,7 +40,7 @@ The :func:`get_annotations` function is the main entry point for
 retrieving annotations. Given a function, class, or module, it returns
 an annotations dictionary in the requested format. This module also provides
 functionality for working directly with the :term:`annotate function`
-that is used to evaluate annotations, such as :func:`get_annotate_function`
+that is used to evaluate annotations, such as :func:`get_annotate_from_class_namespace`
 and :func:`call_annotate_function`, as well as the
 :func:`call_evaluate_function` function for working with
 :term:`evaluate functions <evaluate function>`.
@@ -132,7 +132,7 @@ Classes
 
       Values are real annotation values (as per :attr:`Format.VALUE` format)
       for defined values, and :class:`ForwardRef` proxies for undefined
-      values. Real objects may contain references to, :class:`ForwardRef`
+      values. Real objects may contain references to :class:`ForwardRef`
       proxy objects.
 
    .. attribute:: STRING
@@ -172,14 +172,21 @@ Classes
       :class:`~ForwardRef`. The string may not be exactly equivalent
       to the original source.
 
-   .. method:: evaluate(*, owner=None, globals=None, locals=None, type_params=None)
+   .. method:: evaluate(*, owner=None, globals=None, locals=None, type_params=None, format=Format.VALUE)
 
       Evaluate the forward reference, returning its value.
 
-      This may throw an exception, such as :exc:`NameError`, if the forward
+      If the *format* argument is :attr:`~Format.VALUE` (the default),
+      this method may throw an exception, such as :exc:`NameError`, if the forward
       reference refers to a name that cannot be resolved. The arguments to this
       method can be used to provide bindings for names that would otherwise
-      be undefined.
+      be undefined. If the *format* argument is :attr:`~Format.FORWARDREF`,
+      the method will never throw an exception, but may return a :class:`~ForwardRef`
+      instance. For example, if the forward reference object contains the code
+      ``list[undefined]``, where ``undefined`` is a name that is not defined,
+      evaluating it with the :attr:`~Format.FORWARDREF` format will return
+      ``list[ForwardRef('undefined')]``. If the *format* argument is
+      :attr:`~Format.STRING`, the method will return :attr:`~ForwardRef.__forward_arg__`.
 
       The *owner* parameter provides the preferred mechanism for passing scope
       information to this method. The owner of a :class:`~ForwardRef` is the
@@ -300,15 +307,13 @@ Functions
 
    .. versionadded:: 3.14
 
-.. function:: get_annotate_function(obj)
+.. function:: get_annotate_from_class_namespace(namespace)
 
-   Retrieve the :term:`annotate function` for *obj*. Return :const:`!None`
-   if *obj* does not have an annotate function. *obj* may be a class, function,
-   module, or a namespace dictionary for a class. The last case is useful during
-   class creation, e.g. in the ``__new__`` method of a metaclass.
-
-   This is usually equivalent to accessing the :attr:`~object.__annotate__`
-   attribute of *obj*, but access through this public function is preferred.
+   Retrieve the :term:`annotate function` from a class namespace dictionary *namespace*.
+   Return :const:`!None` if the namespace does not contain an annotate function.
+   This is primarily useful before the class has been fully created (e.g., in a metaclass);
+   after the class exists, the annotate function can be retrieved with ``cls.__annotate__``.
+   See :ref:`below <annotationlib-metaclass>` for an example using this function in a metaclass.
 
    .. versionadded:: 3.14
 
@@ -407,3 +412,76 @@ Functions
 
    .. versionadded:: 3.14
 
+
+Recipes
+-------
+
+.. _annotationlib-metaclass:
+
+Using annotations in a metaclass
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A :ref:`metaclass <metaclasses>` may want to inspect or even modify the annotations
+in a class body during class creation. Doing so requires retrieving annotations
+from the class namespace dictionary. For classes created with
+``from __future__ import annotations``, the annotations will be in the ``__annotations__``
+key of the dictionary. For other classes with annotations,
+:func:`get_annotate_from_class_namespace` can be used to get the
+annotate function, and :func:`call_annotate_function` can be used to call it and
+retrieve the annotations. Using the :attr:`~Format.FORWARDREF` format will usually
+be best, because this allows the annotations to refer to names that cannot yet be
+resolved when the class is created.
+
+To modify the annotations, it is best to create a wrapper annotate function
+that calls the original annotate function, makes any necessary adjustments, and
+returns the result.
+
+Below is an example of a metaclass that filters out all :class:`typing.ClassVar`
+annotations from the class and puts them in a separate attribute:
+
+.. code-block:: python
+
+   import annotationlib
+   import typing
+
+   class ClassVarSeparator(type):
+      def __new__(mcls, name, bases, ns):
+         if "__annotations__" in ns:  # from __future__ import annotations
+            annotations = ns["__annotations__"]
+            classvar_keys = {
+               key for key, value in annotations.items()
+               # Use string comparison for simplicity; a more robust solution
+               # could use annotationlib.ForwardRef.evaluate
+               if value.startswith("ClassVar")
+            }
+            classvars = {key: annotations[key] for key in classvar_keys}
+            ns["__annotations__"] = {
+               key: value for key, value in annotations.items()
+               if key not in classvar_keys
+            }
+            wrapped_annotate = None
+         elif annotate := annotationlib.get_annotate_from_class_namespace(ns):
+            annotations = annotationlib.call_annotate_function(
+               annotate, format=annotationlib.Format.FORWARDREF
+            )
+            classvar_keys = {
+               key for key, value in annotations.items()
+               if typing.get_origin(value) is typing.ClassVar
+            }
+            classvars = {key: annotations[key] for key in classvar_keys}
+
+            def wrapped_annotate(format):
+               annos = annotationlib.call_annotate_function(annotate, format, owner=typ)
+               return {key: value for key, value in annos.items() if key not in classvar_keys}
+
+         else:  # no annotations
+            classvars = {}
+            wrapped_annotate = None
+         typ = super().__new__(mcls, name, bases, ns)
+
+         if wrapped_annotate is not None:
+            # Wrap the original __annotate__ with a wrapper that removes ClassVars
+            typ.__annotate__ = wrapped_annotate
+         typ.classvars = classvars  # Store the ClassVars in a separate attribute
+         return typ
+

Doc/library/os.rst

@@ -2338,6 +2338,7 @@ features:
    This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to
    supply :ref:`paths relative to directory descriptors <dir_fd>`, and :ref:`not
    following symlinks <follow_symlinks>`.
+   The default value of *follow_symlinks* is ``False`` on Windows.
 
    .. audit-event:: os.link src,dst,src_dir_fd,dst_dir_fd os.link
 

Doc/reference/datamodel.rst

@@ -1228,15 +1228,9 @@ Special attributes
        :attr:`__annotations__ attributes <object.__annotations__>`.
 
        For best practices on working with :attr:`~object.__annotations__`,
-       please see :mod:`annotationlib`.
-
-       .. caution::
-
-          Accessing the :attr:`!__annotations__` attribute of a class
-          object directly may yield incorrect results in the presence of
-          metaclasses. In addition, the attribute may not exist for
-          some classes. Use :func:`annotationlib.get_annotations` to
-          retrieve class annotations safely.
+       please see :mod:`annotationlib`. Where possible, use
+       :func:`annotationlib.get_annotations` instead of accessing this
+       attribute directly.
 
        .. versionchanged:: 3.14
           Annotations are now :ref:`lazily evaluated <lazy-evaluation>`.
@@ -1247,13 +1241,6 @@ Special attributes
        if the class has no annotations.
        See also: :attr:`__annotate__ attributes <object.__annotate__>`.
 
-       .. caution::
-
-          Accessing the :attr:`!__annotate__` attribute of a class
-          object directly may yield incorrect results in the presence of
-          metaclasses. Use :func:`annotationlib.get_annotate_function` to
-          retrieve the annotate function safely.
-
        .. versionadded:: 3.14
 
    * - .. attribute:: type.__type_params__

Doc/whatsnew/3.14.rst

@@ -2213,6 +2213,12 @@ Others
   integer must implement either :meth:`~object.__int__` or
   :meth:`~object.__index__`. (Contributed by Mark Dickinson in :gh:`119743`.)
 
+* The default :term:`interactive` shell now supports import autocompletion.
+  This means that typing ``import foo`` and pressing ``<tab>`` will suggest
+  modules starting with ``foo``. Similarly, typing ``from foo import b`` will
+  suggest submodules of ``foo`` starting with ``b``. Note that autocompletion
+  of module attributes is not currently supported.
+  (Contributed by Tomas Roun in :gh:`69605`.)
 
 CPython Bytecode Changes
 ========================

Include/cpython/pystate.h

@@ -194,7 +194,7 @@ struct _ts {
     /* The thread's exception stack entry.  (Always the last entry.) */
     _PyErr_StackItem exc_state;
 
-    PyObject *previous_executor;
+    PyObject *current_executor;
 
     uint64_t dict_global_version;
 

Include/internal/pycore_compile.h

@@ -34,16 +34,16 @@ PyAPI_FUNC(PyCodeObject*) _PyAST_Compile(
     int optimize,
     struct _arena *arena);
 
-/* AST optimizations */
-extern int _PyCompile_AstOptimize(
+/* AST preprocessing */
+extern int _PyCompile_AstPreprocess(
     struct _mod *mod,
     PyObject *filename,
     PyCompilerFlags *flags,
     int optimize,
     struct _arena *arena,
     int syntax_check_only);
 
-extern int _PyAST_Optimize(
+extern int _PyAST_Preprocess(
     struct _mod *,
     struct _arena *arena,
     PyObject *filename,

Include/internal/pycore_interp_structs.h

@@ -923,6 +923,8 @@ struct _is {
     PyObject *common_consts[NUM_COMMON_CONSTANTS];
     bool jit;
     struct _PyExecutorObject *executor_list_head;
+    struct _PyExecutorObject *executor_deletion_list_head;
+    int executor_deletion_list_remaining_capacity;
     size_t trace_run_counter;
     _rare_events rare_events;
     PyDict_WatchCallback builtins_dict_watcher;

Include/internal/pycore_opcode_metadata.h

@@ -69,7 +69,7 @@ typedef struct {
 typedef struct {
     uint32_t target;
     _Py_BackoffCounter temperature;
-    const struct _PyExecutorObject *executor;
+    struct _PyExecutorObject *executor;
 } _PyExitData;
 
 typedef struct _PyExecutorObject {
@@ -84,6 +84,10 @@ typedef struct _PyExecutorObject {
     _PyExitData exits[1];
 } _PyExecutorObject;
 
+/* If pending deletion list gets large enough, then scan,
+ * and free any executors that aren't executing
+ * i.e. any that aren't a thread's current_executor. */
+#define EXECUTOR_DELETE_LIST_MAX 100
 
 // Export for '_opcode' shared extension (JIT compiler).
 PyAPI_FUNC(_PyExecutorObject*) _Py_GetExecutor(PyCodeObject *code, int offset);
@@ -304,6 +308,9 @@ static inline int is_terminator(const _PyUOpInstruction *uop)
 }
 
 PyAPI_FUNC(int) _PyDumpExecutors(FILE *out);
+#ifdef _Py_TIER2
+extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp);
+#endif
 
 #ifdef __cplusplus
 }