license-scan: add latest spdx license data

The latest SPDX data includes the Pixar license, which is a modified
Apache-2.0 license. This rare license often confuses the scanner when it
encounters the common Apache-2.0 license, so a new config option
`spdx.ignore-licenses` has been added to ignore specific licenses from
the input SPDX data.

This change also adds the ability to skip directories when scanning
sources. This is useful for scanning the source directory filled with
sample license texts in the spdx crate itself.

Author: cbgbt

configs/cargo-deny/clarify.toml

@@ -1,3 +1,10 @@
+[spdx]
+ignore-licenses = [
+    # Apache-2.0 is often misclassified as Pixar, which is a significantly more rare
+    # https://github.com/jpeddicord/askalono/issues/94
+    "Pixar"
+]
+
 [clarify.askalono]
 expression = "Apache-2.0"
 license-files = [

configs/cargo-make/clarify.toml

@@ -1,3 +1,10 @@
+[spdx]
+ignore-licenses = [
+    # Apache-2.0 is often misclassified as Pixar, which is a significantly more rare
+    # https://github.com/jpeddicord/askalono/issues/94
+    "Pixar"
+]
+
 [clarify.bstr]
 expression = "(MIT OR Apache-2.0) AND Unicode-DFS-2016"
 license-files = [

license-scan/Cargo.lock

@@ -17,8 +17,9 @@ ignore = "0.4"
 lazy_static = "1"
 semver = { version = "1", features = ["serde"] }
 serde = { version = "1", features = ["derive"] }
-spdx = "0.3"
+spdx = "0.10"
 structopt = { version = "0.3", default-features = false }
+tempfile = "3"
 toml = "0.8"
 twox-hash = "1"
 walkdir = "2"

license-scan/Cargo.toml

@@ -1,3 +1,10 @@
+[spdx]
+ignore-licenses = [
+    # Apache-2.0 is often misclassified as Pixar, which is a significantly more rare
+    # https://github.com/jpeddicord/askalono/issues/94
+    "Pixar"
+]
+
 [clarify.askalono]
 expression = "Apache-2.0"
 license-files = [
@@ -21,7 +28,6 @@ license-files = [
     { path = "COPYING", hash = 0x278afbcf },
     { path = "LICENSE-APACHE", hash = 0x24b54f4b },
     { path = "LICENSE-MIT", hash = 0x462dee44 },
-    { path = "src/unicode/data/LICENSE-UNICODE", hash = 0x70f7339 },
 ]
 
 [clarify.crossbeam-channel]
@@ -58,12 +64,16 @@ license-files = [
     { path = "src/unicode_tables/LICENSE-UNICODE", hash = 0xa7f28b93 },
 ]
 
-[clarify.unicode-ident]
-expression = "(MIT OR Apache-2.0) AND Unicode-DFS-2016"
+[clarify.spdx]
+expression = "MIT OR Apache-2.0"
 license-files = [
-    { path = "LICENSE-APACHE", hash = 0xb5518783 },
-    { path = "LICENSE-MIT", hash = 0x386ca1bc },
-    { path = "LICENSE-UNICODE", hash = 0x9698cbbe },
+    { path = "LICENSE-MIT", hash = 0xa502ee8a },
+    { path = "LICENSE-APACHE", hash = 0x4fccb6b7 },
+]
+skip-dirs = [
+    # The spdx crate contains the full text of referred licenses
+    "src/text/licenses",
+    "src/text/exceptions",
 ]
 
 [clarify.zstd-safe]

license-scan/clarify.toml

@@ -0,0 +1,99 @@
+use anyhow::{ensure, Context, Result};
+use serde::Deserialize;
+use std::collections::HashSet;
+use std::path::Path;
+use walkdir::WalkDir;
+
+#[derive(Debug, Deserialize, Default)]
+#[serde(rename_all = "kebab-case")]
+pub(crate) struct SPDXOptions {
+    /// A set of licenses to ignore from the SPDX license data.
+    #[serde(default)]
+    pub(crate) ignore_licenses: HashSet<String>,
+}
+
+impl SPDXOptions {
+    /// Mogrifies SPDX licenses using a set of static rules.
+    ///
+    /// This function is implemented to work around quirks in the SPDX data that cause unusual behavior
+    /// in askalono during license identification.
+    pub(crate) fn preprocess_licenses(
+        &self,
+        input_path: impl AsRef<Path>,
+        output_path: impl AsRef<Path>,
+    ) -> Result<()> {
+        let input_path = input_path.as_ref();
+        let output_path = output_path.as_ref();
+        ensure!(
+            input_path.is_dir(),
+            "License preprocessing input path is not a directory."
+        );
+        ensure!(
+            output_path.is_dir(),
+            "License preprocessing output path is not a directory."
+        );
+        ensure!(
+            input_path.canonicalize()? != output_path.canonicalize()?,
+            "License preprocessing must write to a new path"
+        );
+
+        let license_iter = WalkDir::new(input_path)
+            .min_depth(1)
+            .max_depth(1)
+            .into_iter();
+
+        for license_file in license_iter {
+            let license_file = license_file?;
+
+            if !license_file.file_type().is_file() {
+                continue;
+            }
+            let license_path = license_file.path();
+            if license_file
+                .file_name()
+                .to_string_lossy()
+                .ends_with(".json")
+                && self.should_include_license_file(license_path)?
+            {
+                let license_output_path = output_path.join(license_file.file_name());
+                std::fs::copy(license_path, &license_output_path).with_context(|| {
+                    format!(
+                        "Failed to copy license from '{}' to '{}'",
+                        license_path.display(),
+                        license_output_path.display()
+                    )
+                })?;
+            }
+        }
+
+        Ok(())
+    }
+
+    fn should_include_license_file(&self, filepath: impl AsRef<Path>) -> Result<bool> {
+        let filepath = filepath.as_ref();
+        let license_name = filepath
+            .file_name()
+            .with_context(|| {
+                format!(
+                    "License file '{}' seemingly has no filename",
+                    filepath.display()
+                )
+            })?
+            .to_str()
+            .with_context(|| {
+                format!(
+                    "License filename '{}' not valid unicode",
+                    filepath.display()
+                )
+            })?
+            .strip_suffix(".json")
+            .with_context(|| {
+                format!(
+                    "License filename '{}' does not end in '.json'",
+                    filepath.display()
+                )
+            })?;
+
+        Ok(!self.ignore_licenses.contains(license_name))
+    }
+}