lowering: Fix captured vars shadowed by an inner global declaration (#57648)

As discovered in #57547, lowering isn't resolving references to captured variables when a global of the same name is declared in any scope in the current function:

```
julia> let
            g = 1
            function f()
                let; global g = 2; end;
                return g    # g is not resolved
            end; f()
        end
ERROR: Found raw symbol in code returned from lowering. Expected all symbols to
have been resolved to GlobalRef or slots.
```

`resolve-scopes` correctly detects that we're returning the local, but scope-blocks are removed in this pass. As a result, `analyze-vars-lambda` finds more globals than `resolve-scopes` did when calling `find-global-decls` (which does not peek inside nested scopes) on `f`.  `analyze-vars-lambda` then calculates the set of captured variables in `f` to be:

```
captvars = (current_env ∩ free_vars) - (new_staticparams ∪ wrong_globals)
```

so `g` in this case is never captured.

This bug was introduced (revealed, maybe) in #57051---the whole resolution step used to happen after lowering, so lowering passes disagreeing on scopes would be less visible.

Fix: omit globals from the free variable list in `analyze-vars-lambda` in the first place.  This way we don't need to call the scope-block-dependent `find-global-decls`.

Author: mlechu

doc/src/manual/variables-and-scoping.md

@@ -136,12 +136,14 @@ inside of another local scope, the scope it creates is nested inside of all the
 local scopes that it appears within, which are all ultimately nested inside of
 the global scope of the module in which the code is evaluated. Variables in
 outer scopes are visible from any scope they contain — meaning that they can be
-read and written in inner scopes — unless there is a local variable with the
-same name that "shadows" the outer variable of the same name. This is true even
-if the outer local is declared after (in the sense of textually below) an inner
+read and written in inner scopes — unless there is a variable with the same name
+that "shadows" the outer variable of the same name. This is true even if the
+outer local is declared after (in the sense of textually below) an inner
 block. When we say that a variable "exists" in a given scope, this means that a
 variable by that name exists in any of the scopes that the current scope is
-nested inside of, including the current one.
+nested inside of, including the current one. If a variable's value is used in a
+local scope, but nothing with its name exists in this scope, it is assumed to be
+a global.
 
 Some programming languages require explicitly declaring new variables before
 using them. Explicit declaration works in Julia too: in any local scope, writing

src/julia-syntax.scm

@@ -3357,11 +3357,11 @@
 (define (lambda-all-vars e)
   (append (lam:argnames e) (caddr e)))
 
-;; compute set of variables referenced in a lambda but not bound by it
+;; compute set of non-global variables referenced in a lambda but not bound by it
 (define (free-vars- e tab)
   (cond ((or (eq? e UNUSED) (underscore-symbol? e)) tab)
         ((symbol? e) (put! tab e #t))
-        ((and (pair? e) (eq? (car e) 'globalref)) tab)
+        ((and (pair? e) (memq (car e) '(global globalref))) tab)
         ((and (pair? e) (eq? (car e) 'break-block)) (free-vars- (caddr e) tab))
         ((and (pair? e) (eq? (car e) 'with-static-parameters)) (free-vars- (cadr e) tab))
         ((or (atom? e) (quoted? e)) tab)
@@ -3385,9 +3385,15 @@
               vi)
     tab))
 
+;; env:      list of vinfo (includes any closure #self#; should not include globals)
+;; captvars: list of vinfo
+;; sp:       list of symbol
+;; new-sp:   list of symbol (static params declared here)
+;; methsig:  `(call (core svec) ...)
+;; tab:      table of (name . var-info)
 (define (analyze-vars-lambda e env captvars sp new-sp methsig tab)
   (let* ((args (lam:args e))
-         (locl (caddr e))
+         (locl (lam:vinfo e))
          (allv (nconc (map arg-name args) locl))
          (fv   (let* ((fv (diff (free-vars (lam:body e)) allv))
                       ;; add variables referenced in declared types for free vars
@@ -3397,27 +3403,23 @@
                                             fv))))
                  (append (diff dv fv) fv)))
          (sig-fv (if methsig (free-vars methsig) '()))
-         (glo  (find-global-decls (lam:body e)))
          ;; make var-info records for vars introduced by this lambda
          (vi   (nconc
                 (map (lambda (decl) (make-var-info (decl-var decl)))
                      args)
                 (map make-var-info locl)))
-         (capt-sp (filter (lambda (v) (or (and (memq v fv) (not (memq v glo)) (not (memq v new-sp)))
+         (capt-sp (filter (lambda (v) (or (and (memq v fv) (not (memq v new-sp)))
                                           (memq v sig-fv)))
                           sp))
          ;; captured vars: vars from the environment that occur
          ;; in our set of free variables (fv).
          (cv    (append (filter (lambda (v) (and (memq (vinfo:name v) fv)
-                                                 (not (memq (vinfo:name v) new-sp))
-                                                 (not (memq (vinfo:name v) glo))))
+                                                 (not (memq (vinfo:name v) new-sp))))
                                 env)
                         (map make-var-info capt-sp)))
          (new-env (append vi
                           ;; new environment: add our vars
-                          (filter (lambda (v)
-                                    (and (not (memq (vinfo:name v) allv))
-                                         (not (memq (vinfo:name v) glo))))
+                          (filter (lambda (v) (not (memq (vinfo:name v) allv)))
                                   env))))
     (analyze-vars (lam:body e)
                   new-env
@@ -4043,7 +4045,7 @@ f(x) = yt(x)
        ((atom? e) e)
        (else
         (case (car e)
-          ((quote top core globalref thismodule lineinfo line break inert module toplevel null true false meta import using) e)
+          ((quote top core global globalref thismodule lineinfo line break inert module toplevel null true false meta import using) e)
           ((toplevel-only)
            ;; hack to avoid generating a (method x) expr for struct types
            (if (eq? (cadr e) 'struct)

test/syntax.jl

@@ -2417,15 +2417,6 @@ macro a35391(b)
 end
 @test @a35391(0) === (0,)
 
-# global declarations from the top level are not inherited by functions.
-# don't allow such a declaration to override an outer local, since it's not
-# clear what it should do.
-@test Meta.lower(Main, :(let
-                           x = 1
-                           let
-                             global x
-                           end
-                         end)) == Expr(:error, "`global x`: x is a local variable in its enclosing scope")
 # note: this `begin` block must be at the top level
 _temp_33553 = begin
     global _x_this_remains_undefined
@@ -2437,6 +2428,70 @@ end
 @test _temp_33553 == 2
 @test !@isdefined(_x_this_remains_undefined)
 
+module GlobalContainment
+using Test
+@testset "scope of global declarations" begin
+
+    # global declarations from the top level are not inherited by functions.
+    # don't allow such a declaration to override an outer local, since it's not
+    # clear what it should do.
+    @test Meta.lower(
+        Main,
+        :(let
+              x = 1
+              let
+                  global x
+              end
+          end)) == Expr(:error, "`global x`: x is a local variable in its enclosing scope")
+
+    # a declared global can shadow a local in an outer scope
+    @test let
+        function f()
+            g0 = 2
+            let; global g0 = 1; end
+            a = () -> (global g0 = 1); a();
+            return g0
+        end
+        (f(), g0);
+    end === (2, 1)
+    @test let
+        function f()
+            let; global g2 = 1; end;
+            let; try; g2 = 2; catch _; end; end;
+        end
+        (f(), g2)
+    end === (2, 1)
+
+    # an inner global declaration should not interfere with the closure (#57547)
+    @test let
+        g3 = 1
+        function f()
+            let; global g3 = 2; end;
+            return g3
+        end
+        f()
+    end === 1
+    @test_throws UndefVarError let
+        function returns_global()
+            for i in 1
+                global ge = 2
+            end
+            return ge # local declared below
+        end
+        ge = returns_global()
+    end
+    @test let
+        function f(x::T) where T
+            function g(x)
+                let; global T = 1; end
+                x::T
+            end; g(x)
+        end; f(1)
+    end === 1
+
+end
+end
+
 # lowering of adjoint
 @test (1 + im)' == 1 - im
 x = let var"'"(x) = 2x