remove_unused_loads

Summary:
The plan is to replace `arc f` buildifer with:
* load removal implemented in starlark
* code formatting with black

This diff implements unused load removal. As a function in `starlark` crate for now.

It does not work perfectly yet. One known issue is this:

For the program:

```
load("foo", "x", "y")
print(x)
```

It produces:

```
load("foo", "x", )
print(x)
```

and black for some mysterious reason reformats this code as:

```
load(
    "foo",
    "x",
)
print(x)
```

instead of placing load on one line.

This can be addressed later if we proceed with black.

# libCST

Note initially I planned to use libCST, but I think load removal is easy enough to implement it just by text manipulation, while implementing with libCST would require less-trivial communication of unused imports between rust and python code.

Reviewed By: ianlevesque

Differential Revision: D48941531

fbshipit-source-id: b4ba870300b8c32828a95f2cf800b10555e4a1cd

Author: stepancheg

starlark/src/analysis/mod.rs

@@ -23,6 +23,7 @@ pub use lint_message::LintMessage;
 pub use types::EvalMessage;
 pub use types::EvalSeverity;
 pub use types::Lint;
+pub use unused_loads::remove::remove_unused_loads;
 
 use crate::analysis::types::LintT;
 use crate::syntax::AstModule;
@@ -36,6 +37,7 @@ mod names;
 mod performance;
 mod types;
 mod underscore;
+mod unused_loads;
 
 /// Run the linter.
 pub trait AstModuleLint {

starlark/src/analysis/unused_loads/find.rs

@@ -0,0 +1,179 @@
+/*
+ * Copyright 2019 The Starlark in Rust Authors.
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use std::collections::HashMap;
+
+use anyhow::Context;
+use dupe::Dupe;
+use starlark_syntax::codemap::CodeMap;
+use starlark_syntax::codemap::FileSpanRef;
+use starlark_syntax::codemap::Spanned;
+use starlark_syntax::slice_vec_ext::SliceExt;
+use starlark_syntax::syntax::ast::LoadArgP;
+use starlark_syntax::syntax::ast::LoadP;
+use starlark_syntax::syntax::ast::StmtP;
+use starlark_syntax::syntax::top_level_stmts::top_level_stmts;
+
+use crate::environment::names::MutableNames;
+use crate::eval::compiler::scope::payload::CstPayload;
+use crate::eval::compiler::scope::scope_resolver_globals::ScopeResolverGlobals;
+use crate::eval::compiler::scope::BindingId;
+use crate::eval::compiler::scope::ModuleScopes;
+use crate::eval::compiler::scope::ResolvedIdent;
+use crate::eval::compiler::scope::Slot;
+use crate::syntax::AstModule;
+use crate::syntax::Dialect;
+use crate::values::FrozenHeap;
+
+/// Unused load statement.
+pub(crate) struct UnusedLoad {
+    /// Location of the statement (i.e. position of `load` keyword).
+    pub(crate) load: Spanned<LoadP<CstPayload>>,
+    /// Unused local names, e. g. `x` in `load("foo", x="y")`.
+    pub(crate) unused_args: Vec<LoadArgP<CstPayload>>,
+}
+
+impl UnusedLoad {
+    /// If the whole `load` statement is unused.
+    pub(crate) fn all_unused(&self) -> bool {
+        self.unused_args.len() == self.load.args.len()
+    }
+}
+
+/// Check if there are `@unused` markers on the lines with the given span.
+fn has_unused_marker_in_range(span: FileSpanRef) -> bool {
+    let begin_line = span.file.find_line(span.span.begin());
+    let end_line = span.file.find_line(span.span.end());
+    for line_no in begin_line..=end_line {
+        let line = span.file.source_line(line_no);
+        if line.ends_with("@unused") || line.contains("@unused ") {
+            return true;
+        }
+    }
+    false
+}
+
+/// Parse the module and find unused loads.
+pub(crate) fn find_unused_loads(
+    name: &str,
+    program: &str,
+) -> anyhow::Result<(CodeMap, Vec<UnusedLoad>)> {
+    let module = AstModule::parse(name, program.to_owned(), &Dialect::Extended)?;
+    let names = MutableNames::new();
+    let heap = FrozenHeap::new();
+    let codemap = heap.alloc_any_display_from_type_name(module.codemap);
+    let module_scopes = ModuleScopes::check_module_err(
+        &names,
+        &heap,
+        &HashMap::new(),
+        module.statement,
+        ScopeResolverGlobals::unknown(),
+        codemap,
+        &module.dialect,
+    )?;
+
+    let mut loads = Vec::new();
+
+    struct LoadSymbol<'a> {
+        arg: &'a LoadArgP<CstPayload>,
+        binding_id: BindingId,
+        used: bool,
+    }
+
+    struct LoadWip<'a> {
+        load: Spanned<&'a LoadP<CstPayload>>,
+        args: Vec<LoadSymbol<'a>>,
+    }
+
+    impl<'a> LoadWip<'a> {
+        fn any_unused(&self) -> bool {
+            self.args.iter().any(|arg| !arg.used)
+        }
+    }
+
+    for top in top_level_stmts(&module_scopes.cst) {
+        if let StmtP::Load(load) = &**top {
+            let args = load.args.try_map(|arg| {
+                anyhow::Ok(LoadSymbol {
+                    arg,
+                    binding_id: arg.local.payload.context("payload is not set")?,
+                    used: false,
+                })
+            })?;
+            loads.push(LoadWip {
+                load: Spanned {
+                    span: top.span,
+                    node: load,
+                },
+                args,
+            });
+        }
+    }
+
+    for top in top_level_stmts(&module_scopes.cst) {
+        top.visit_ident(|ident| {
+            println!("visit ident: {:?}", ident);
+            let ResolvedIdent::Slot(Slot::Module(_), binding_id) = ident
+                .payload
+                .context("ident is not resolved (internal error)")?
+            else {
+                return Ok(());
+            };
+            for load in &mut loads {
+                for arg in &mut load.args {
+                    if arg.binding_id == binding_id {
+                        arg.used = true;
+                    }
+                }
+            }
+            anyhow::Ok(())
+        })?;
+    }
+
+    let mut unused = Vec::new();
+
+    for load in loads {
+        if !load.any_unused() {
+            continue;
+        }
+        let unused_args: Vec<_> = load
+            .args
+            .into_iter()
+            .filter_map(|arg| {
+                if arg.used {
+                    None
+                } else if has_unused_marker_in_range(FileSpanRef {
+                    file: &codemap,
+                    span: arg.arg.span_with_trailing_comma(),
+                }) {
+                    None
+                } else {
+                    Some(arg.arg.clone())
+                }
+            })
+            .collect();
+        if unused_args.is_empty() {
+            continue;
+        }
+        unused.push(UnusedLoad {
+            load: load.load.map(|load| load.clone()),
+            unused_args,
+        });
+    }
+
+    Ok(((*codemap).dupe(), unused))
+}

starlark/src/analysis/unused_loads/find/one_of_two_unused.golden

@@ -0,0 +1,18 @@
+# @generated
+# To regenerate, run:
+# ```
+# STARLARK_RUST_REGENERATE_GOLDEN_TESTS=1 cargo test -p starlark --lib tests
+# ```
+
+Program:
+load("foo", "x", "y")
+print(x)
+
+Unused loads:
+
+error: Unused load
+ --> one_of_two_unused:1:18
+  |
+1 | load("foo", "x", "y")
+  |                  ^^^
+  |

starlark/src/analysis/unused_loads/find/simple.golden

@@ -0,0 +1,17 @@
+# @generated
+# To regenerate, run:
+# ```
+# STARLARK_RUST_REGENERATE_GOLDEN_TESTS=1 cargo test -p starlark --lib tests
+# ```
+
+Program:
+load("foo", "x")
+
+Unused loads:
+
+error: Unused load
+ --> simple:1:1
+  |
+1 | load("foo", "x")
+  | ^^^^^^^^^^^^^^^^
+  |

starlark/src/analysis/unused_loads/find/unused_annotation_on_arg.golden

@@ -0,0 +1,20 @@
+# @generated
+# To regenerate, run:
+# ```
+# STARLARK_RUST_REGENERATE_GOLDEN_TESTS=1 cargo test -p starlark --lib tests
+# ```
+
+Program:
+load("foo",
+    "T", # @unused
+    "U",
+     )
+
+Unused loads:
+
+error: Unused load
+ --> unused_annotation_on_arg:3:5
+  |
+3 |     "U",
+  |     ^^^
+  |

starlark/src/analysis/unused_loads/find/used_in_top_level_assignment.golden

@@ -0,0 +1,11 @@
+# @generated
+# To regenerate, run:
+# ```
+# STARLARK_RUST_REGENERATE_GOLDEN_TESTS=1 cargo test -p starlark --lib tests
+# ```
+
+Program:
+load("foo", "x")
+y = x
+
+No unused loads

starlark/src/analysis/unused_loads/find/used_in_type_expr.golden

@@ -0,0 +1,16 @@
+# @generated
+# To regenerate, run:
+# ```
+# STARLARK_RUST_REGENERATE_GOLDEN_TESTS=1 cargo test -p starlark --lib tests
+# ```
+
+Program:
+load("foo", "T")
+load("bar", "U")
+
+y: T = 1
+
+def f(x: U):
+    pass
+
+No unused loads

starlark/src/analysis/unused_loads/find/with_rename.golden

@@ -0,0 +1,18 @@
+# @generated
+# To regenerate, run:
+# ```
+# STARLARK_RUST_REGENERATE_GOLDEN_TESTS=1 cargo test -p starlark --lib tests
+# ```
+
+Program:
+load("foo", x="y", z="w")
+y = z
+
+Unused loads:
+
+error: Unused load
+ --> with_rename:1:13
+  |
+1 | load("foo", x="y", z="w")
+  |             ^^^^^
+  |