Skip to content

Commit 2250fa6

Browse files
ntBrentBre
authored
Fix syntax error false positives for await outside functions (#21763)
## Summary Fixes #21750 and a related bug in `PLE1142`. We were not properly considering generators to be valid `await` contexts, which caused the `F704` issue. One of the tests I added for this also uncovered an issue in `PLE1142` for comprehensions nested within async generators because we were only checking the current scope rather than traversing the nested context. ## Test Plan Both of these rules are implemented as semantic syntax errors, so I added tests (and fixes) in both Ruff and ty.
1 parent 392a8e4 commit 2250fa6

File tree

9 files changed

+147
-16
lines changed

9 files changed

+147
-16
lines changed

‎crates/ruff_linter/resources/test/fixtures/pyflakes/F704.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,24 @@ def _():
1717

1818
# Valid yield scope
1919
yield 3
20+
21+
22+
# await is valid in any generator, sync or async
23+
(await cor async for cor in f()) # ok
24+
(await cor for cor in f()) # ok
25+
26+
# but not in comprehensions
27+
[await cor async for cor in f()] # F704
28+
{await cor async for cor in f()} # F704
29+
{await cor: 1 async for cor in f()} # F704
30+
[await cor for cor in f()] # F704
31+
{await cor for cor in f()} # F704
32+
{await cor: 1 for cor in f()} # F704
33+
34+
# or in the iterator of an async generator, which is evaluated in the parent
35+
# scope
36+
(cor async for cor in await f()) # F704
37+
(await cor async for cor in [await c for c in f()]) # F704
38+
39+
# this is also okay because the comprehension is within the generator scope
40+
([await c for c in cor] async for cor in f()) # ok

‎crates/ruff_linter/resources/test/fixtures/syntax_errors/await_outside_async_function.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,5 @@ def func():
33

44
# Top-level await
55
await 1
6+
7+
([await c for c in cor] async for cor in func()) # ok

‎crates/ruff_linter/src/checkers/ast/mod.rs‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -780,6 +780,10 @@ impl SemanticSyntaxContext for Checker<'_> {
780780
match scope.kind {
781781
ScopeKind::Class(_) => return false,
782782
ScopeKind::Function(_) | ScopeKind::Lambda(_) => return true,
783+
ScopeKind::Generator {
784+
kind: GeneratorKind::Generator,
785+
..
786+
} => return true,
783787
ScopeKind::Generator { .. }
784788
| ScopeKind::Module
785789
| ScopeKind::Type
@@ -829,14 +833,19 @@ impl SemanticSyntaxContext for Checker<'_> {
829833
self.source_type.is_ipynb()
830834
}
831835

832-
fn in_generator_scope(&self) -> bool {
833-
matches!(
834-
&self.semantic.current_scope().kind,
835-
ScopeKind::Generator {
836-
kind: GeneratorKind::Generator,
837-
..
836+
fn in_generator_context(&self) -> bool {
837+
for scope in self.semantic.current_scopes() {
838+
if matches!(
839+
scope.kind,
840+
ScopeKind::Generator {
841+
kind: GeneratorKind::Generator,
842+
..
843+
}
844+
) {
845+
return true;
838846
}
839-
)
847+
}
848+
false
840849
}
841850

842851
fn in_loop_context(&self) -> bool {

‎crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__F704_F704.py.snap‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,88 @@ F704 `await` statement outside of a function
3737
12 |
3838
13 | def _():
3939
|
40+
41+
F704 `await` statement outside of a function
42+
--> F704.py:27:2
43+
|
44+
26 | # but not in comprehensions
45+
27 | [await cor async for cor in f()] # F704
46+
| ^^^^^^^^^
47+
28 | {await cor async for cor in f()} # F704
48+
29 | {await cor: 1 async for cor in f()} # F704
49+
|
50+
51+
F704 `await` statement outside of a function
52+
--> F704.py:28:2
53+
|
54+
26 | # but not in comprehensions
55+
27 | [await cor async for cor in f()] # F704
56+
28 | {await cor async for cor in f()} # F704
57+
| ^^^^^^^^^
58+
29 | {await cor: 1 async for cor in f()} # F704
59+
30 | [await cor for cor in f()] # F704
60+
|
61+
62+
F704 `await` statement outside of a function
63+
--> F704.py:29:2
64+
|
65+
27 | [await cor async for cor in f()] # F704
66+
28 | {await cor async for cor in f()} # F704
67+
29 | {await cor: 1 async for cor in f()} # F704
68+
| ^^^^^^^^^
69+
30 | [await cor for cor in f()] # F704
70+
31 | {await cor for cor in f()} # F704
71+
|
72+
73+
F704 `await` statement outside of a function
74+
--> F704.py:30:2
75+
|
76+
28 | {await cor async for cor in f()} # F704
77+
29 | {await cor: 1 async for cor in f()} # F704
78+
30 | [await cor for cor in f()] # F704
79+
| ^^^^^^^^^
80+
31 | {await cor for cor in f()} # F704
81+
32 | {await cor: 1 for cor in f()} # F704
82+
|
83+
84+
F704 `await` statement outside of a function
85+
--> F704.py:31:2
86+
|
87+
29 | {await cor: 1 async for cor in f()} # F704
88+
30 | [await cor for cor in f()] # F704
89+
31 | {await cor for cor in f()} # F704
90+
| ^^^^^^^^^
91+
32 | {await cor: 1 for cor in f()} # F704
92+
|
93+
94+
F704 `await` statement outside of a function
95+
--> F704.py:32:2
96+
|
97+
30 | [await cor for cor in f()] # F704
98+
31 | {await cor for cor in f()} # F704
99+
32 | {await cor: 1 for cor in f()} # F704
100+
| ^^^^^^^^^
101+
33 |
102+
34 | # or in the iterator of an async generator, which is evaluated in the parent
103+
|
104+
105+
F704 `await` statement outside of a function
106+
--> F704.py:36:23
107+
|
108+
34 | # or in the iterator of an async generator, which is evaluated in the parent
109+
35 | # scope
110+
36 | (cor async for cor in await f()) # F704
111+
| ^^^^^^^^^
112+
37 | (await cor async for cor in [await c for c in f()]) # F704
113+
|
114+
115+
F704 `await` statement outside of a function
116+
--> F704.py:37:30
117+
|
118+
35 | # scope
119+
36 | (cor async for cor in await f()) # F704
120+
37 | (await cor async for cor in [await c for c in f()]) # F704
121+
| ^^^^^^^
122+
38 |
123+
39 | # this is also okay because the comprehension is within the generator scope
124+
|

‎crates/ruff_linter/src/snapshots/ruff_linter__linter__tests__await_outside_async_function.py.snap‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,6 @@ PLE1142 `await` should be used within an async function
1717
4 | # Top-level await
1818
5 | await 1
1919
| ^^^^^^^
20+
6 |
21+
7 | ([await c for c in cor] async for cor in func()) # ok
2022
|

‎crates/ruff_python_parser/src/semantic_errors.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -896,7 +896,7 @@ impl SemanticSyntaxChecker {
896896
// This check is required in addition to avoiding calling this function in `visit_expr`
897897
// because the generator scope applies to nested parts of the `Expr::Generator` that are
898898
// visited separately.
899-
if ctx.in_generator_scope() {
899+
if ctx.in_generator_context() {
900900
return;
901901
}
902902
Self::add_error(
@@ -2096,11 +2096,11 @@ pub trait SemanticSyntaxContext {
20962096
/// Returns `true` if the visitor is in a function scope.
20972097
fn in_function_scope(&self) -> bool;
20982098

2099-
/// Returns `true` if the visitor is in a generator scope.
2099+
/// Returns `true` if the visitor is within a generator scope.
21002100
///
21012101
/// Note that this refers to an `Expr::Generator` precisely, not to comprehensions more
21022102
/// generally.
2103-
fn in_generator_scope(&self) -> bool;
2103+
fn in_generator_context(&self) -> bool;
21042104

21052105
/// Returns `true` if the source file is a Jupyter notebook.
21062106
fn in_notebook(&self) -> bool;

‎crates/ruff_python_parser/tests/fixtures.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -573,7 +573,7 @@ impl SemanticSyntaxContext for SemanticSyntaxCheckerVisitor<'_> {
573573
true
574574
}
575575

576-
fn in_generator_scope(&self) -> bool {
576+
fn in_generator_context(&self) -> bool {
577577
true
578578
}
579579

‎crates/ty_python_semantic/resources/mdtest/diagnostics/semantic_syntax_errors.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,10 @@ await C()
143143
def f():
144144
# error: [invalid-syntax] "`await` outside of an asynchronous function"
145145
await C()
146+
147+
(await cor async for cor in f()) # ok
148+
(await cor for cor in f()) # ok
149+
([await c for c in cor] async for cor in f()) # ok
146150
```
147151

148152
Generators are evaluated lazily, so `await` is allowed, even outside of a function.

‎crates/ty_python_semantic/src/semantic_index/builder.rs‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2845,6 +2845,11 @@ impl SemanticSyntaxContext for SemanticIndexBuilder<'_, '_> {
28452845
match scope.kind() {
28462846
ScopeKind::Class => return false,
28472847
ScopeKind::Function | ScopeKind::Lambda => return true,
2848+
ScopeKind::Comprehension
2849+
if matches!(scope.node(), NodeWithScopeKind::GeneratorExpression(_)) =>
2850+
{
2851+
return true;
2852+
}
28482853
ScopeKind::Comprehension
28492854
| ScopeKind::Module
28502855
| ScopeKind::TypeAlias
@@ -2894,11 +2899,14 @@ impl SemanticSyntaxContext for SemanticIndexBuilder<'_, '_> {
28942899
matches!(kind, ScopeKind::Function | ScopeKind::Lambda)
28952900
}
28962901

2897-
fn in_generator_scope(&self) -> bool {
2898-
matches!(
2899-
self.scopes[self.current_scope()].node(),
2900-
NodeWithScopeKind::GeneratorExpression(_)
2901-
)
2902+
fn in_generator_context(&self) -> bool {
2903+
for scope_info in &self.scope_stack {
2904+
let scope = &self.scopes[scope_info.file_scope_id];
2905+
if matches!(scope.node(), NodeWithScopeKind::GeneratorExpression(_)) {
2906+
return true;
2907+
}
2908+
}
2909+
false
29022910
}
29032911

29042912
fn in_notebook(&self) -> bool {

0 commit comments

Comments
 (0)