Package and export configuration files from plugins (#2044)

This change allows plugins to package and declare exported configuration
files using the syntax below:

```toml
[plugins.definitions.some-plugin]
# Paths are relative to this TOML file location (plugin.toml, source.toml, etc)
exported_config_paths = ["relative/path/to/config.yml", "other/config.alternate.yml"]

# ...
```

Declaring an exported configuration allows a configuration file to be
automatically included at runtime as a fallback default if no other
configuration files are present or detected by the workspace (including
`.qlty/configs` directory). The configuration file will be symlinked
just like any other configuration.

### Disabling Export Behavior from a Downstream Consumer

Users who want to opt out of automatic importing of configuration files
can either:

1. Define a default configuration file in their workspace source tree
(or `.qlty/configs`), or,
2. Override the `plugins.definitions.<plugin>.exported_config_paths` as:

```toml
[plugins.definitions.some-plugin]
exported_config_paths = []
```

### Using with Sources

This behavior is can used with custom sources to automatically include
custom configuration from a given layered source:

#### \> .qlty/qlty.toml

```toml
[[source]]
name = "default"
default = true

[[source]]
name = "configs"
directory = "./configuration" # this can also be a repository
```

#### \> configuration/source.toml

```toml
[plugins.definitions.rubocop]
exported_config_paths = ["rubocop/.rubocop.yml"]
```

#### \> configuration/rubocop/.rubocop.yml

```yaml
AllCops:
  Enable: true
# ...
```

Any project including the above source will automatically receive custom
rubocop configuration that will automatically be applied when running
`qlty check`.

Author: lsegal

qlty-check/src/executor.rs

@@ -29,6 +29,7 @@ use qlty_types::analysis::v1::{Issue, Message, MessageLevel};
 use rand::seq::SliceRandom;
 use rand::thread_rng;
 use rayon::prelude::*;
+use staging_area::load_config_file_from_source;
 use std::collections::{HashMap, HashSet};
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicUsize, Ordering};
@@ -291,6 +292,14 @@ impl Executor {
             }
         }
 
+        let exported_config_paths = self
+            .plan
+            .invocations
+            .iter()
+            .flat_map(|invocation| invocation.plugin.exported_config_paths.clone())
+            .unique()
+            .collect_vec();
+
         debug!(
             "Walker found {} config and affects cache files in {:.2}s",
             repository_config_files.len(),
@@ -318,7 +327,7 @@ impl Executor {
             if self.plan.workspace.root != self.plan.staging_area.destination_directory {
                 // for formatters
                 let loaded_config_file = load_config_file_from_qlty_dir(
-                    &PathBuf::from(config_file),
+                    config_file,
                     &self.plan.workspace,
                     &self.plan.staging_area.destination_directory,
                 )?;
@@ -330,7 +339,7 @@ impl Executor {
 
             // for linters
             let loaded_config_file = load_config_file_from_qlty_dir(
-                &PathBuf::from(config_file),
+                config_file,
                 &self.plan.workspace,
                 &self.plan.workspace.root,
             )?;
@@ -340,6 +349,28 @@ impl Executor {
             }
         }
 
+        for config_file in &exported_config_paths {
+            if self.plan.workspace.root != self.plan.staging_area.destination_directory {
+                // for formatters
+                let loaded_config_file = load_config_file_from_source(
+                    config_file,
+                    &self.plan.staging_area.destination_directory,
+                )?;
+
+                if !loaded_config_file.is_empty() {
+                    loaded_config_files.push(loaded_config_file);
+                }
+            }
+
+            // for linters
+            let loaded_config_file =
+                load_config_file_from_source(&config_file, &self.plan.workspace.root)?;
+
+            if !loaded_config_file.is_empty() {
+                loaded_config_files.push(loaded_config_file);
+            }
+        }
+
         debug!(
             "Staged {} config files in {:.2}s",
             repository_config_files.len(),

qlty-check/src/executor/staging_area.rs

@@ -218,86 +218,71 @@ impl StagingArea {
 }
 
 pub fn load_config_file_from_repository(
-    config_file: &Path,
+    config_file: impl AsRef<Path>,
     workspace: &Workspace,
-    destination: &Path,
-) -> Result<()> {
-    let to = destination.join(config_file.strip_prefix(&workspace.root).unwrap());
-
-    if to.exists() {
-        debug!("Config file already exists in workspace: {:?}", to);
-        return Ok(());
-    }
-
-    let to_dir = to.parent();
-
-    if !to_dir.unwrap().exists() {
-        debug!("Creating destination dir: {:?}", destination);
-        create_dir_all(to_dir.unwrap()).with_context(|| {
-            format!(
-                "Failed to create workspace entries destination dir: {}",
-                destination.display()
-            )
-        })?;
-    }
+    destination: impl AsRef<Path>,
+) -> Result<String> {
+    let to = destination
+        .as_ref()
+        .join(config_file.as_ref().strip_prefix(&workspace.root).unwrap());
 
-    debug!(
-        "Copying config file from repository: {:?} -> {:?}",
-        config_file, to
-    );
-    copy(config_file, to.clone()).with_context(|| {
-        format!(
-            "Failed to copy config file {} to {}",
-            config_file.display(),
-            to.display()
-        )
-    })?;
+    load_config_file_from(config_file, to)
+}
 
-    Ok(())
+pub fn load_config_file_from_source(
+    config_file: impl AsRef<Path>,
+    destination: impl AsRef<Path>,
+) -> Result<String> {
+    load_config_file_from(
+        &config_file,
+        destination
+            .as_ref()
+            .join(config_file.as_ref().file_name().unwrap()),
+    )
 }
 
 pub fn load_config_file_from_qlty_dir(
-    config_file: &Path,
+    config_file: impl AsRef<Path>,
     workspace: &Workspace,
-    destination: &Path,
+    destination: impl AsRef<Path>,
 ) -> Result<String> {
-    let config_file_name = config_file.file_name().unwrap();
+    let config_file_name = config_file.as_ref().file_name().unwrap();
     let from = workspace.library()?.configs_dir().join(config_file_name);
+    let to = destination.as_ref().join(config_file_name);
+    load_config_file_from(from, to)
+}
 
-    if from.exists() {
-        let to = destination.join(config_file_name);
-        if to.exists() {
-            debug!("Config file already exists in workspace: {:?}", to);
-            return Ok(to.display().to_string());
-        }
+fn load_config_file_from(from: impl AsRef<Path>, to: impl AsRef<Path>) -> Result<String> {
+    let from = from.as_ref();
+    let to = to.as_ref();
+    if !from.exists() {
+        return Ok("".to_string());
+    } else if to.exists() {
+        debug!("Config file already exists in workspace: {:?}", to);
+        return Ok(to.display().to_string());
+    }
 
-        debug!(
-            "Symlinking config file from qlty dir: {:?} -> {:?}",
-            from, to
-        );
+    debug!("Symlinking config file: {:?} -> {:?}", from, to);
 
-        let result: std::io::Result<_>;
-        #[cfg(windows)]
-        {
-            result = std::os::windows::fs::symlink_file(from.clone(), to.clone());
-        }
-        #[cfg(unix)]
-        {
-            result = std::os::unix::fs::symlink(from.clone(), to.clone());
-        }
+    symlink(from, to).with_context(|| {
+        format!(
+            "Failed to symlink config file {} to {}",
+            from.display(),
+            to.display()
+        )
+    })?;
 
-        result.with_context(|| {
-            format!(
-                "Failed to symlink config file {} to {}",
-                from.display(),
-                to.display()
-            )
-        })?;
+    Ok(to.display().to_string())
+}
 
-        return Ok(to.display().to_string());
-    }
+#[cfg(windows)]
+fn symlink(from: &Path, to: &Path) -> std::io::Result<()> {
+    std::os::windows::fs::symlink_file(from, to)
+}
 
-    Ok("".to_string())
+#[cfg(unix)]
+fn symlink(from: &Path, to: &Path) -> std::io::Result<()> {
+    std::os::unix::fs::symlink(from, to)
 }
 
 #[cfg(test)]
@@ -387,4 +372,107 @@ mod test {
         let clone = stage.clone();
         assert_eq!(clone.read("test".into()).unwrap(), "expected");
     }
+
+    #[test]
+    fn test_load_config_file_from_repository() {
+        let (_, paths) = new_staging_area(Mode::ReadWrite);
+
+        let config_file = paths.source.path().join("abc").join("conf.yml");
+        create_dir_all(config_file.parent().unwrap()).unwrap();
+        create_dir_all(paths.dest.path().join("abc")).unwrap();
+        std::fs::write(&config_file, "repository_config_content").unwrap();
+
+        let workspace = Workspace::for_root(paths.source.path()).unwrap();
+        let result = load_config_file_from_repository(&config_file, &workspace, paths.dest.path());
+        assert!(result.is_ok());
+
+        let dest_file = paths.dest.path().join("abc").join("conf.yml");
+        assert!(dest_file.exists());
+
+        let content = std::fs::read_to_string(&dest_file).unwrap();
+        assert_eq!(content, "repository_config_content");
+    }
+
+    #[test]
+    fn test_load_config_file_from_symlink_fail() {
+        let (_, paths) = new_staging_area(Mode::ReadWrite);
+
+        let config_file = paths.source.path().join("abc").join("conf.yml");
+        create_dir_all(config_file.parent().unwrap()).unwrap();
+
+        std::fs::write(&config_file, "repository_config_content").unwrap();
+
+        let workspace = Workspace::for_root(paths.source.path()).unwrap();
+        let result = load_config_file_from_repository(&config_file, &workspace, paths.dest.path());
+        assert!(result.is_err());
+
+        let dest_file = paths.dest.path().join("abc").join("conf.yml");
+        assert!(!dest_file.exists());
+    }
+
+    #[test]
+    fn test_load_config_file_from_source() {
+        let (_, paths) = new_staging_area(Mode::ReadWrite);
+
+        let config_file = paths.source.path().join("nested").join("conf.yml");
+        create_dir_all(config_file.parent().unwrap()).unwrap();
+        std::fs::write(&config_file, "source_config_content").unwrap();
+
+        let result = load_config_file_from_source(&config_file, paths.dest.path());
+        assert!(result.is_ok());
+
+        let dest_file = paths.dest.path().join("conf.yml");
+        assert!(dest_file.exists());
+
+        let content = std::fs::read_to_string(&dest_file).unwrap();
+        assert_eq!(content, "source_config_content");
+    }
+
+    #[test]
+    fn test_load_config_file_from_qlty_dir() {
+        let (_, paths) = new_staging_area(Mode::ReadWrite);
+        let mock_workspace_path = tempdir().unwrap();
+        let configs_dir = mock_workspace_path.path().join(".qlty").join("configs");
+        create_dir_all(&configs_dir).unwrap();
+
+        let config_file = configs_dir.join("conf.yml");
+        std::fs::write(&config_file, "qlty_dir_config_content").unwrap();
+
+        let workspace = Workspace::for_root(mock_workspace_path.path()).unwrap();
+        let result = load_config_file_from_qlty_dir("conf.yml", &workspace, paths.dest.path());
+        assert!(result.is_ok());
+
+        let dest_file = paths.dest.path().join("conf.yml");
+        assert!(dest_file.exists());
+
+        let content = std::fs::read_to_string(&dest_file).unwrap();
+        assert_eq!(content, "qlty_dir_config_content");
+    }
+
+    #[test]
+    fn test_load_config_file_nonexistent() {
+        let (_, paths) = new_staging_area(Mode::ReadWrite);
+        let nonexistent_file = paths.source.path().join("conf.yml");
+        let result = load_config_file_from(&nonexistent_file, paths.dest.path().join("conf.yml"));
+
+        assert!(result.is_ok());
+        assert_eq!(result.unwrap(), "");
+    }
+
+    #[test]
+    fn test_load_config_file_already_exists() {
+        let (_, paths) = new_staging_area(Mode::ReadWrite);
+        let source_file = paths.source.path().join("conf.yml");
+        let dest_file = paths.dest.path().join("conf.yml");
+
+        std::fs::write(&source_file, "source_content").unwrap();
+        std::fs::write(&dest_file, "destination_content").unwrap();
+
+        let result = load_config_file_from(&source_file, &dest_file);
+        assert!(result.is_ok());
+        assert_eq!(result.unwrap(), dest_file.display().to_string());
+
+        let content = std::fs::read_to_string(&dest_file).unwrap();
+        assert_eq!(content, "destination_content");
+    }
 }

qlty-cli/tests/cmd/check/exported_config_files.in/.gitignore

@@ -0,0 +1,4 @@
+.qlty/results
+.qlty/logs
+.qlty/out
+.qlty/sources

qlty-cli/tests/cmd/check/exported_config_files.in/.qlty/qlty.toml

@@ -0,0 +1,9 @@
+config_version = "0"
+
+[[source]]
+name = "config-exporter"
+directory = "config-exporter"
+
+[[plugin]]
+name = "config-exporter"
+version = "1.0.0"

qlty-cli/tests/cmd/check/exported_config_files.in/config-exporter/config.yml

@@ -0,0 +1 @@
+configuration: true

qlty-cli/tests/cmd/check/exported_config_files.in/config-exporter/results.txt

@@ -0,0 +1 @@
+sample.sh:2:1: ERROR invalid line

qlty-cli/tests/cmd/check/exported_config_files.in/config-exporter/source.toml

@@ -0,0 +1,29 @@
+config_version = "0"
+
+[plugins.definitions.config-exporter]
+file_types = ["shell"]
+config_files = ["config.yml", "config.alternate.yml", "results.txt"]
+exported_config_paths = [
+  "config.yml",
+  "results.txt",
+  "subdir/config.alternate.yml",
+]
+
+[plugins.definitions.config-exporter.drivers.lint]
+prepare_script = "mkdir ${linter} && echo dir %2 > ${linter}/ls.cmd || echo dir %2 > ${linter}/ls.cmd"
+script = "ls -l config.yml"
+success_codes = [0]
+output = "pass_fail"
+
+[plugins.definitions.config-exporter.drivers.lint2]
+prepare_script = "mkdir ${linter} && echo dir %2 > ${linter}/ls.cmd || echo dir %2 > ${linter}/ls.cmd"
+script = "ls -l config.alternate.yml"
+success_codes = [0]
+output = "pass_fail"
+
+[plugins.definitions.config-exporter.drivers.lint3]
+prepare_script = "mkdir ${linter} && echo type %1 > ${linter}/cat.cmd || echo type %1 > ${linter}/cat.cmd"
+script = "cat results.txt"
+output_format = "regex"
+output_regex = "((?P<path>.*):(?P<line>-?\\d+):(?P<col>-?\\d+): (?P<code>\\S+) (?P<message>.+))\n"
+success_codes = [0]

qlty-cli/tests/cmd/check/exported_config_files.in/config-exporter/subdir/config.alternate.yml

@@ -0,0 +1 @@
+configuration: true

qlty-cli/tests/cmd/check/exported_config_files.in/results.txt

@@ -0,0 +1 @@
+sample.sh:2:1: OVERRIDE valid line

qlty-cli/tests/cmd/check/exported_config_files.in/sample.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo hi