Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
1c691ea
Lazy imports grammar / AST changes
DinoV Sep 16, 2025
3b0d745
Add __lazy_import__, check sys.modules before import
DinoV Sep 18, 2025
9eef03c
Export lazy_import in imp module
DinoV Sep 18, 2025
de281fd
Add lazy import filter
DinoV Sep 19, 2025
41ab092
Add compatiblity mode
DinoV Sep 22, 2025
058bc6e
Flow import func through to lazy imports object and __lazy_import__
DinoV Sep 22, 2025
6d7c87a
Reify lazy objects when accessed via the module object
DinoV Sep 22, 2025
f992ee7
Import lazy.get
DinoV Sep 22, 2025
6a91132
Implement better error
pablogsal Sep 22, 2025
03a419a
Fix reference and global dict in spezializing LOAD_GLOBAL
pablogsal Sep 23, 2025
e0878be
Remove copyright
pablogsal Sep 23, 2025
f9880bf
More fixes
pablogsal Sep 23, 2025
f3f5795
Fix recursive lazy imports and error path in bytecodes.c
pablogsal Sep 23, 2025
44a3e46
fix offset in addr2line
pablogsal Sep 23, 2025
73de8d0
Add global flag
pablogsal Sep 25, 2025
07a633f
Draft: force * imports to be eager
pablogsal Sep 25, 2025
20b14d9
Syntax restrictions for lazy imports
pablogsal Sep 25, 2025
164423b
Fix submodules crash
pablogsal Sep 26, 2025
00e7800
Implement disabling imports in try/except and * imports, report error…
DinoV Sep 29, 2025
781eedb
Add PyExc_ImportCycleError and raise it when a cycle is detected
DinoV Sep 29, 2025
9be59ec
Publish lazy imported packages on parent
DinoV Sep 29, 2025
b179da2
Re-enable eagerly returning imported module
DinoV Sep 29, 2025
9078f57
Fix __lazy_modules__
pablogsal Oct 1, 2025
f67310c
Add sys.set_lazy_imports_filter
pablogsal Oct 1, 2025
39c33df
Simplify grammar
pablogsal Oct 2, 2025
c8c8838
Move __lazy_imports__ check into the interpreter
DinoV Oct 2, 2025
7c49405
Don't allow __lazy_imports__ to work in try/except
DinoV Oct 2, 2025
5d6026a
Make imports in with blocks syntax errors
pablogsal Oct 2, 2025
c0c0d80
Highlight lazy imports in the new REPL
bswck Oct 3, 2025
5ff0dd2
Merge pull request #2 from bswck/lazy-import-pyrepl-highlight
pablogsal Oct 3, 2025
214b254
Fix global membership in LOAD_NAME
pablogsal Oct 7, 2025
e5e9592
C was a mistake
pablogsal Oct 7, 2025
aa85f9d
dir() doesn't reify module
DinoV Oct 7, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions Grammar/python.gram
Original file line number Diff line number Diff line change
Expand Up @@ -121,9 +121,9 @@ simple_stmts[asdl_stmt_seq*]:
simple_stmt[stmt_ty] (memo):
| assignment
| &"type" type_alias
| &('import' | 'from' | "lazy" ) import_stmt
| e=star_expressions { _PyAST_Expr(e, EXTRA) }
| &'return' return_stmt
| &('import' | 'from') import_stmt
| &'raise' raise_stmt
| &'pass' pass_stmt
| &'del' del_stmt
Expand Down Expand Up @@ -216,21 +216,23 @@ assert_stmt[stmt_ty]:
| invalid_assert_stmt
| 'assert' a=expression b=[',' z=expression { z }] { _PyAST_Assert(a, b, EXTRA) }

import_stmt[stmt_ty]:
import_stmt[stmt_ty](memo):
| invalid_import
| import_name
| import_from

# Import statements
# -----------------

import_name[stmt_ty]: 'import' a=dotted_as_names { _PyAST_Import(a, EXTRA) }
import_name[stmt_ty]:
| lazy="lazy"? 'import' a=dotted_as_names { _PyAST_Import(a, lazy ? 1 : 0, EXTRA) }

# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
import_from[stmt_ty]:
| 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets {
_PyPegen_checked_future_import(p, b->v.Name.id, c, _PyPegen_seq_count_dots(a), EXTRA) }
| 'from' a=('.' | '...')+ 'import' b=import_from_targets {
_PyAST_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), EXTRA) }
| lazy="lazy"? 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets {
_PyPegen_checked_future_import(p, b->v.Name.id, c, _PyPegen_seq_count_dots(a), lazy ? 1 : 0, EXTRA) }
| lazy="lazy"? 'from' a=('.' | '...')+ 'import' b=import_from_targets {
_PyAST_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), lazy ? 1 : 0, EXTRA) }
import_from_targets[asdl_alias_seq*]:
| '(' a=import_from_as_names [','] ')' { a }
| import_from_as_names !','
Expand Down
1 change: 1 addition & 0 deletions Include/cpython/initconfig.h
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,7 @@ typedef struct PyConfig {
int enable_gil;
int tlbc_enabled;
#endif
int lazy_imports;

/* --- Path configuration inputs ------------ */
int pathconfig_warnings;
Expand Down
10 changes: 10 additions & 0 deletions Include/import.h
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,16 @@ PyAPI_FUNC(int) PyImport_AppendInittab(
PyObject* (*initfunc)(void)
);

typedef enum {
PyLazyImportsMode_Default,
PyLazyImportsMode_ForcedOff,
PyLazyImportsMode_ForcedOn,
} PyImport_LazyImportsMode;

PyAPI_FUNC(int) PyImport_SetLazyImports(PyImport_LazyImportsMode mode, PyObject *filter);

PyAPI_FUNC(PyImport_LazyImportsMode) PyImport_LazyImportsEnabled(void);

#ifndef Py_LIMITED_API
# define Py_CPYTHON_IMPORT_H
# include "cpython/import.h"
Expand Down
11 changes: 7 additions & 4 deletions Include/internal/pycore_ast.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Include/internal/pycore_ast_state.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 8 additions & 1 deletion Include/internal/pycore_ceval.h
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,14 @@ PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc,
PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg);
PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs);
PyAPI_FUNC(PyObject *) _PyEval_ImportFrom(PyThreadState *, PyObject *, PyObject *);
PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *);
PyAPI_FUNC(PyObject *) _PyEval_LazyImportName(PyThreadState *tstate, PyObject *builtins, PyObject *globals,
PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level, int lazy);
PyAPI_FUNC(PyObject *) _PyEval_LazyImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name);
PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *tstate, PyObject *builtins, PyObject *globals, PyObject *locals,
PyObject *name, PyObject *fromlist, PyObject *level);
PyObject *
_PyEval_ImportNameWithImport(PyThreadState *tstate, PyObject *import_func, PyObject *globals, PyObject *locals,
PyObject *name, PyObject *fromlist, PyObject *level);
PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs);
PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys);
PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr);
Expand Down
2 changes: 2 additions & 0 deletions Include/internal/pycore_global_objects_fini_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Include/internal/pycore_global_strings.h
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,8 @@ struct _Py_global_strings {
STRUCT_FOR_ID(__iter__)
STRUCT_FOR_ID(__itruediv__)
STRUCT_FOR_ID(__ixor__)
STRUCT_FOR_ID(__lazy_import__)
STRUCT_FOR_ID(__lazy_modules__)
STRUCT_FOR_ID(__le__)
STRUCT_FOR_ID(__len__)
STRUCT_FOR_ID(__length_hint__)
Expand Down
16 changes: 16 additions & 0 deletions Include/internal/pycore_import.h
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,18 @@ extern int _PyImport_FixupBuiltin(
PyObject *modules
);

extern PyObject *
_PyImport_ResolveName(PyThreadState *tstate, PyObject *name, PyObject *globals, int level);
extern PyObject *
_PyImport_GetAbsName(PyThreadState *tstate, PyObject *name, PyObject *globals, int level);
extern PyObject *
_PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import);
extern PyObject *
_PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, PyObject *name, PyObject *builtins, PyObject *globals,
PyObject *locals, PyObject *fromlist,
int level);


#ifdef HAVE_DLOPEN
# include <dlfcn.h> // RTLD_NOW, RTLD_LAZY
# if HAVE_DECL_RTLD_NOW
Expand Down Expand Up @@ -73,6 +85,10 @@ extern int _PyImport_IsDefaultImportFunc(
PyInterpreterState *interp,
PyObject *func);

extern int _PyImport_IsDefaultLazyImportFunc(
PyInterpreterState *interp,
PyObject *func);

extern PyObject * _PyImport_GetImportlibLoader(
PyInterpreterState *interp,
const char *loader_name);
Expand Down
5 changes: 5 additions & 0 deletions Include/internal/pycore_interp_structs.h
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,11 @@ struct _import_state {
int dlopenflags;
#endif
PyObject *import_func;
PyObject *lazy_import_func;
int lazy_imports_mode;
PyObject *lazy_imports_filter;
PyObject *lazy_importing_modules;
PyObject *lazy_modules;
/* The global import lock. */
_PyRecursiveMutex lock;
/* diagnostic info in PyImport_ImportModuleLevelObject() */
Expand Down
31 changes: 31 additions & 0 deletions Include/internal/pycore_lazyimportobject.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/* File added for Lazy Imports */

/* Lazy object interface */

#ifndef Py_LAZYIMPORTOBJECT_H
#define Py_LAZYIMPORTOBJECT_H
#ifdef __cplusplus
extern "C" {
#endif

PyAPI_DATA(PyTypeObject) PyLazyImport_Type;
#define PyLazyImport_CheckExact(op) Py_IS_TYPE((op), &PyLazyImport_Type)

typedef struct {
PyObject_HEAD
PyObject *lz_import_func;
PyObject *lz_from;
PyObject *lz_attr;
/* Frame information for the original import location */
PyCodeObject *lz_code; /* code object where the lazy import was created */
int lz_instr_offset; /* instruction offset where the lazy import was created */
} PyLazyImportObject;


PyAPI_FUNC(PyObject *) _PyLazyImport_GetName(PyObject *lazy_import);
PyAPI_FUNC(PyObject *) _PyLazyImport_New(PyObject *import_func, PyObject *from, PyObject *attr);

#ifdef __cplusplus
}
#endif
#endif /* !Py_LAZYIMPORTOBJECT_H */
3 changes: 2 additions & 1 deletion Include/internal/pycore_magic_number.h
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,7 @@ Known values:
Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST)
Python 3.15a1 3654 (Fix missing exception handlers in logical expression)
Python 3.15a1 3655 (Fix miscompilation of some module-level annotations)
Python 3.15a1 3656 Lazy imports IMPORT_NAME opcode changes


Python 3.16 will start with 3700
Expand All @@ -299,7 +300,7 @@ PC/launcher.c must also be updated.

*/

#define PYC_MAGIC_NUMBER 3655
#define PYC_MAGIC_NUMBER 3657
/* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes
(little-endian) and then appending b'\r\n'. */
#define PYC_MAGIC_NUMBER_TOKEN \
Expand Down
2 changes: 2 additions & 0 deletions Include/internal/pycore_moduleobject.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ typedef struct {
PyObject *md_weaklist;
// for logging purposes after md_dict is cleared
PyObject *md_name;
// module version we last checked for lazy values
uint32_t m_dict_version;
#ifdef Py_GIL_DISABLED
void *md_gil;
#endif
Expand Down
2 changes: 2 additions & 0 deletions Include/internal/pycore_runtime_init_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Include/internal/pycore_symtable.h
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,8 @@ typedef struct _symtable_entry {
unsigned ste_method : 1; /* true if block is a function block defined in class scope */
unsigned ste_has_conditional_annotations : 1; /* true if block has conditionally executed annotations */
unsigned ste_in_conditional_block : 1; /* set while we are inside a conditionally executed block */
unsigned ste_in_try_block : 1; /* set while we are inside a try/except block */
unsigned ste_in_with_block : 1; /* set while we are inside a with block */
unsigned ste_in_unevaluated_annotation : 1; /* set while we are processing an annotation that will not be evaluated */
int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */
_Py_SourceLocation ste_loc; /* source location of block */
Expand Down
8 changes: 8 additions & 0 deletions Include/internal/pycore_unicodeobject_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions Include/pyerrors.h
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,9 @@ PyAPI_DATA(PyObject *) PyExc_EOFError;
PyAPI_DATA(PyObject *) PyExc_FloatingPointError;
PyAPI_DATA(PyObject *) PyExc_OSError;
PyAPI_DATA(PyObject *) PyExc_ImportError;
#if !defined(Py_LIMITED_API)
PyAPI_DATA(PyObject *) PyExc_ImportCycleError;
#endif
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03060000
PyAPI_DATA(PyObject *) PyExc_ModuleNotFoundError;
#endif
Expand Down
1 change: 1 addition & 0 deletions Lib/_compat_pickle.py
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,7 @@
REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'OSError')

PYTHON3_IMPORTERROR_EXCEPTIONS = (
'ImportCycleError',
'ModuleNotFoundError',
)

Expand Down
2 changes: 2 additions & 0 deletions Lib/_pyrepl/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,8 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool:
TI(T.NAME, string=s)
):
return not keyword.iskeyword(s)
case (None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT), TI(string="lazy"), TI(string="import")):
return True
case _:
return False

Expand Down
7 changes: 7 additions & 0 deletions Lib/dis.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure', 'annotate')

ENTER_EXECUTOR = opmap['ENTER_EXECUTOR']
IMPORT_NAME = opmap['IMPORT_NAME']
LOAD_GLOBAL = opmap['LOAD_GLOBAL']
LOAD_SMALL_INT = opmap['LOAD_SMALL_INT']
BINARY_OP = opmap['BINARY_OP']
Expand Down Expand Up @@ -601,6 +602,12 @@ def get_argval_argrepr(self, op, arg, offset):
argval, argrepr = _get_name_info(arg//4, get_name)
if (arg & 1) and argrepr:
argrepr = f"{argrepr} + NULL|self"
elif deop == IMPORT_NAME:
argval, argrepr = _get_name_info(arg//4, get_name)
if (arg & 1) and argrepr:
argrepr = f"{argrepr} + lazy"
elif (arg & 2) and argrepr:
argrepr = f"{argrepr} + eager"
else:
argval, argrepr = _get_name_info(arg, get_name)
elif deop in hasjump or deop in hasexc:
Expand Down
2 changes: 2 additions & 0 deletions Lib/importlib/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@

from ._bootstrap import __import__

from _imp import lazy_import, set_lazy_imports


def invalidate_caches():
"""Call the invalidate_caches() method on all meta path finders stored in
Expand Down
6 changes: 6 additions & 0 deletions Lib/importlib/_bootstrap.py
Original file line number Diff line number Diff line change
Expand Up @@ -1359,6 +1359,12 @@ def _find_and_load_unlocked(name, import_):
except AttributeError:
msg = f"Cannot set an attribute on {parent!r} for child module {child!r}"
_warnings.warn(msg, ImportWarning)
# Set attributes to lazy submodules on the module.
try:
_imp._set_lazy_attributes(module, name)
except Exception as e:
msg = f"Cannot set lazy attributes on {name!r}: {e!r}"
_warnings.warn(msg, ImportWarning)
return module


Expand Down
1 change: 1 addition & 0 deletions Lib/keyword.py

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Lib/test/exception_hierarchy.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ BaseException
├── EOFError
├── ExceptionGroup [BaseExceptionGroup]
├── ImportError
│ └── ImportCycleError
│ └── ModuleNotFoundError
├── LookupError
│ ├── IndexError
Expand Down
8 changes: 4 additions & 4 deletions Lib/test/test_ast/data/ast_repr.txt
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,10 @@ Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='ex
Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[])
Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=None)], type_ignores=[])
Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=Constant(value='message', kind=None))], type_ignores=[])
Module(body=[Import(names=[alias(name='sys', asname=None)])], type_ignores=[])
Module(body=[Import(names=[alias(name='foo', asname='bar')])], type_ignores=[])
Module(body=[ImportFrom(module='sys', names=[alias(name='x', asname='y')], level=0)], type_ignores=[])
Module(body=[ImportFrom(module='sys', names=[alias(name='v', asname=None)], level=0)], type_ignores=[])
Module(body=[Import(names=[alias(name='sys', asname=None)], is_lazy=0)], type_ignores=[])
Module(body=[Import(names=[alias(name='foo', asname='bar')], is_lazy=0)], type_ignores=[])
Module(body=[ImportFrom(module='sys', names=[alias(name='x', asname='y')], level=0, is_lazy=0)], type_ignores=[])
Module(body=[ImportFrom(module='sys', names=[alias(name='v', asname=None)], level=0, is_lazy=0)], type_ignores=[])
Module(body=[Global(names=['v'])], type_ignores=[])
Module(body=[Expr(value=Constant(value=1, kind=None))], type_ignores=[])
Module(body=[Pass()], type_ignores=[])
Expand Down
4 changes: 2 additions & 2 deletions Lib/test/test_ast/test_ast.py
Original file line number Diff line number Diff line change
Expand Up @@ -1722,8 +1722,8 @@ def check_text(code, empty, full, **kwargs):

check_text(
"import _ast as ast; from module import sub",
empty="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)])",
full="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)], type_ignores=[])",
empty="Module(body=[Import(names=[alias(name='_ast', asname='ast')], is_lazy=0), ImportFrom(module='module', names=[alias(name='sub')], level=0, is_lazy=0)])",
full="Module(body=[Import(names=[alias(name='_ast', asname='ast')], is_lazy=0), ImportFrom(module='module', names=[alias(name='sub')], level=0, is_lazy=0)], type_ignores=[])",
)

def test_copy_location(self):
Expand Down
Loading
Loading