Merge pull request #174 from github/analyze-command

Author: hendrikvanantwerpen

tree-sitter-stack-graphs/CHANGELOG.md

@@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Library
+
+#### Added
+
+- A new `CancelAfterDuration` implementation of `CancellationFlag` that cancels the computation after a certain amount of time.
+
+#### Changed
+
+- The `LanguageConfiguration::matches_file` method takes a `ContentProvider` instead of an `Option<&str>` value. This allows lazy file reading *after* the filename is checked, instead of the unconditional loading required before. To give content readers the opportunity to cache read values, a mutable reference is required. The return type has changed to `std::io::Result` to propagate possible errors from content providers. A `FileReader` implementation that caches the last read file is provided as well.
+
+### CLI
+
+#### Added
+
+- A new `analyze` command that computes stack graphs and partial paths for all given source files and directories. The command does not produce any output at the moment. Analysis per file can be limited using the `--max-file-time` flag.
+
 ## v0.6.0 -- 2023-01-13
 
 ### Library

tree-sitter-stack-graphs/src/cli.rs

@@ -55,6 +55,7 @@
 
 pub(self) const MAX_PARSE_ERRORS: usize = 5;
 
+pub mod analyze;
 pub mod init;
 pub mod load;
 pub mod parse;
@@ -64,13 +65,15 @@ mod util;
 pub mod path_loading {
     use clap::Subcommand;
 
+    use crate::cli::analyze::AnalyzeArgs;
     use crate::cli::init::InitArgs;
     use crate::cli::load::PathLoaderArgs;
     use crate::cli::parse::ParseArgs;
     use crate::cli::test::TestArgs;
 
     #[derive(Subcommand)]
     pub enum Subcommands {
+        Analyze(Analyze),
         Init(Init),
         Parse(Parse),
         Test(Test),
@@ -79,13 +82,30 @@ pub mod path_loading {
     impl Subcommands {
         pub fn run(&self) -> anyhow::Result<()> {
             match self {
+                Self::Analyze(cmd) => cmd.run(),
                 Self::Init(cmd) => cmd.run(),
                 Self::Parse(cmd) => cmd.run(),
                 Self::Test(cmd) => cmd.run(),
             }
         }
     }
 
+    /// Analyze command
+    #[derive(clap::Parser)]
+    pub struct Analyze {
+        #[clap(flatten)]
+        load_args: PathLoaderArgs,
+        #[clap(flatten)]
+        analyze_args: AnalyzeArgs,
+    }
+
+    impl Analyze {
+        pub fn run(&self) -> anyhow::Result<()> {
+            let mut loader = self.load_args.get()?;
+            self.analyze_args.run(&mut loader)
+        }
+    }
+
     /// Init command
     #[derive(clap::Parser)]
     pub struct Init {
@@ -135,27 +155,45 @@ pub mod path_loading {
 pub mod provided_languages {
     use clap::Subcommand;
 
+    use crate::cli::analyze::AnalyzeArgs;
+    use crate::cli::load::LanguageConfigurationsLoaderArgs;
     use crate::cli::parse::ParseArgs;
     use crate::cli::test::TestArgs;
     use crate::loader::LanguageConfiguration;
 
-    use super::load::LanguageConfigurationsLoaderArgs;
-
     #[derive(Subcommand)]
     pub enum Subcommands {
+        Analyze(Analyze),
         Parse(Parse),
         Test(Test),
     }
 
     impl Subcommands {
         pub fn run(&self, configurations: Vec<LanguageConfiguration>) -> anyhow::Result<()> {
             match self {
+                Self::Analyze(cmd) => cmd.run(configurations),
                 Self::Parse(cmd) => cmd.run(configurations),
                 Self::Test(cmd) => cmd.run(configurations),
             }
         }
     }
 
+    /// Analyze command
+    #[derive(clap::Parser)]
+    pub struct Analyze {
+        #[clap(flatten)]
+        load_args: LanguageConfigurationsLoaderArgs,
+        #[clap(flatten)]
+        analyze_args: AnalyzeArgs,
+    }
+
+    impl Analyze {
+        pub fn run(&self, configurations: Vec<LanguageConfiguration>) -> anyhow::Result<()> {
+            let mut loader = self.load_args.get(configurations)?;
+            self.analyze_args.run(&mut loader)
+        }
+    }
+
     /// Parse command
     #[derive(clap::Parser)]
     pub struct Parse {

tree-sitter-stack-graphs/src/cli/analyze.rs

@@ -0,0 +1,195 @@
+// -*- coding: utf-8 -*-
+// ------------------------------------------------------------------------------------------------
+// Copyright © 2021, stack-graphs authors.
+// Licensed under either of Apache License, Version 2.0, or MIT license, at your option.
+// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details.
+// ------------------------------------------------------------------------------------------------
+
+use anyhow::anyhow;
+use anyhow::Context as _;
+use clap::Args;
+use clap::ValueHint;
+use stack_graphs::graph::StackGraph;
+use stack_graphs::partial::PartialPaths;
+use stack_graphs::stitching::Database;
+use std::collections::HashMap;
+use std::path::Path;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::Duration;
+use tree_sitter_graph::Variables;
+use walkdir::WalkDir;
+
+use crate::cli::util::duration_from_seconds_str;
+use crate::cli::util::map_parse_errors;
+use crate::cli::util::path_exists;
+use crate::loader::FileReader;
+use crate::loader::Loader;
+use crate::CancelAfterDuration;
+use crate::CancellationFlag;
+use crate::LoadError;
+use crate::NoCancellation;
+
+use super::util::FileStatusLogger;
+
+/// Analyze sources
+#[derive(Args)]
+pub struct AnalyzeArgs {
+    /// Source file or directory paths.
+    #[clap(
+        value_name = "SOURCE_PATH",
+        required = true,
+        value_hint = ValueHint::AnyPath,
+        parse(from_os_str),
+        validator_os = path_exists,
+    )]
+    pub source_paths: Vec<PathBuf>,
+
+    #[clap(long, short = 'v')]
+    pub verbose: bool,
+
+    /// Maximum runtime per file in seconds.
+    #[clap(
+        long,
+        value_name = "SECONDS",
+        parse(try_from_str = duration_from_seconds_str),
+    )]
+    pub max_file_time: Option<Duration>,
+}
+
+impl AnalyzeArgs {
+    pub fn new(source_paths: Vec<PathBuf>) -> Self {
+        Self {
+            source_paths,
+            verbose: false,
+            max_file_time: None,
+        }
+    }
+
+    pub fn run(&self, loader: &mut Loader) -> anyhow::Result<()> {
+        for source_path in &self.source_paths {
+            if source_path.is_dir() {
+                let source_root = source_path;
+                for source_entry in WalkDir::new(source_root)
+                    .follow_links(true)
+                    .into_iter()
+                    .filter_map(|e| e.ok())
+                    .filter(|e| e.file_type().is_file())
+                {
+                    let source_path = source_entry.path();
+                    self.analyze_file_with_context(source_root, source_path, loader)?;
+                }
+            } else {
+                let source_root = source_path.parent().unwrap();
+                self.analyze_file_with_context(source_root, source_path, loader)?;
+            }
+        }
+        Ok(())
+    }
+
+    /// Analyze file and add error context to any failures that are returned.
+    fn analyze_file_with_context(
+        &self,
+        source_root: &Path,
+        source_path: &Path,
+        loader: &mut Loader,
+    ) -> anyhow::Result<()> {
+        self.analyze_file(source_root, source_path, loader)
+            .with_context(|| format!("Error analyzing file {}", source_path.display()))
+    }
+
+    fn analyze_file(
+        &self,
+        source_root: &Path,
+        source_path: &Path,
+        loader: &mut Loader,
+    ) -> anyhow::Result<()> {
+        let mut file_status = FileStatusLogger::new(source_path, self.verbose);
+
+        let mut file_reader = FileReader::new();
+        let lc = match loader.load_for_file(source_path, &mut file_reader, &NoCancellation) {
+            Ok(Some(sgl)) => sgl,
+            Ok(None) => return Ok(()),
+            Err(crate::loader::LoadError::Cancelled(_)) => {
+                file_status.warn("language loading timed out")?;
+                return Ok(());
+            }
+            Err(e) => return Err(e.into()),
+        };
+        let source = file_reader.get(source_path)?;
+
+        let mut cancellation_flag: Arc<dyn CancellationFlag> = Arc::new(NoCancellation);
+        if let Some(max_file_time) = self.max_file_time {
+            cancellation_flag = CancelAfterDuration::new(max_file_time);
+        }
+
+        file_status.processing()?;
+
+        let mut graph = StackGraph::new();
+        let file = match graph.add_file(&source_path.to_string_lossy()) {
+            Ok(file) => file,
+            Err(_) => return Err(anyhow!("Duplicate file {}", source_path.display())),
+        };
+
+        let relative_source_path = source_path.strip_prefix(source_root).unwrap();
+        let result = if let Some(fa) = source_path
+            .file_name()
+            .and_then(|f| lc.special_files.get(&f.to_string_lossy()))
+        {
+            fa.build_stack_graph_into(
+                &mut graph,
+                file,
+                &relative_source_path,
+                &source,
+                &mut std::iter::empty(),
+                &HashMap::new(),
+                cancellation_flag.as_ref(),
+            )
+        } else {
+            let globals = Variables::new();
+            lc.sgl.build_stack_graph_into(
+                &mut graph,
+                file,
+                &source,
+                &globals,
+                cancellation_flag.as_ref(),
+            )
+        };
+        match result {
+            Err(LoadError::ParseErrors(parse_errors)) => {
+                let parse_error = map_parse_errors(source_path, &parse_errors, &source, "");
+                file_status.error("parsing failed")?;
+                eprintln!("{}", parse_error);
+                return Ok(());
+            }
+            Err(LoadError::Cancelled(_)) => {
+                file_status.warn("parsing timed out")?;
+                return Ok(());
+            }
+            Err(e) => return Err(e.into()),
+            Ok(_) => {}
+        };
+
+        let mut partials = PartialPaths::new();
+        let mut db = Database::new();
+        match partials.find_all_partial_paths_in_file(
+            &graph,
+            file,
+            &cancellation_flag.as_ref(),
+            |g, ps, p| {
+                if p.is_complete_as_possible(g) {
+                    db.add_partial_path(g, ps, p);
+                }
+            },
+        ) {
+            Ok(_) => {}
+            Err(_) => {
+                file_status.warn("path computation timed out")?;
+                return Ok(());
+            }
+        }
+
+        file_status.ok("success")?;
+        Ok(())
+    }
+}

tree-sitter-stack-graphs/src/cli/parse.rs

@@ -15,6 +15,7 @@ use tree_sitter::Parser;
 use tree_sitter_graph::parse_error::ParseError;
 
 use crate::cli::util::path_exists;
+use crate::loader::FileReader;
 use crate::loader::Loader;
 use crate::LoadError;
 
@@ -34,11 +35,12 @@ impl ParseArgs {
     }
 
     fn parse_file(&self, file_path: &Path, loader: &mut Loader) -> anyhow::Result<()> {
-        let source = std::fs::read_to_string(file_path)?;
-        let lang = match loader.load_tree_sitter_language_for_file(file_path, Some(&source))? {
+        let mut file_reader = FileReader::new();
+        let lang = match loader.load_tree_sitter_language_for_file(file_path, &mut file_reader)? {
             Some(sgl) => sgl,
             None => return Err(anyhow!("No stack graph language found")),
         };
+        let source = file_reader.get(file_path)?;
 
         let mut parser = Parser::new();
         parser.set_language(lang)?;

tree-sitter-stack-graphs/src/cli/test.rs

@@ -24,6 +24,7 @@ use walkdir::WalkDir;
 use crate::cli::util::map_parse_errors;
 use crate::cli::util::path_exists;
 use crate::cli::util::PathSpec;
+use crate::loader::FileReader;
 use crate::loader::LanguageConfiguration;
 use crate::loader::Loader;
 use crate::test::Test;
@@ -193,8 +194,8 @@ impl TestArgs {
         test_path: &Path,
         loader: &mut Loader,
     ) -> anyhow::Result<TestResult> {
-        let source = std::fs::read_to_string(test_path)?;
-        let lc = match loader.load_for_file(test_path, Some(&source), &NoCancellation)? {
+        let mut file_reader = FileReader::new();
+        let lc = match loader.load_for_file(test_path, &mut file_reader, &NoCancellation)? {
             Some(sgl) => sgl,
             None => {
                 if self.show_ignored {
@@ -203,6 +204,7 @@ impl TestArgs {
                 return Ok(TestResult::new());
             }
         };
+        let source = file_reader.get(test_path)?;
         let default_fragment_path = test_path.strip_prefix(test_root).unwrap();
         let mut test = Test::from_source(&test_path, &source, default_fragment_path)?;
         self.load_builtins_into(&lc, &mut test.graph)
@@ -224,7 +226,10 @@ impl TestArgs {
                     &test_fragment.globals,
                     &NoCancellation,
                 )?;
-            } else if lc.matches_file(&test_fragment.path, Some(&test_fragment.source)) {
+            } else if lc.matches_file(
+                &test_fragment.path,
+                &mut Some(test_fragment.source.as_ref()),
+            )? {
                 globals.clear();
                 test_fragment.add_globals_to(&mut globals);
                 self.build_fragment_stack_graph_into(
@@ -281,7 +286,7 @@ impl TestArgs {
     ) -> anyhow::Result<()> {
         match sgl.build_stack_graph_into(graph, file, source, globals, &NoCancellation) {
             Err(LoadError::ParseErrors(parse_errors)) => {
-                Err(map_parse_errors(test_path, &parse_errors, source))
+                Err(map_parse_errors(test_path, &parse_errors, source, "  "))
             }
             Err(e) => Err(e.into()),
             Ok(_) => Ok(()),