Better identify repo licenses

This commit introduces calls to GitHub and GitLab APIs to better
identify licenses in the Git repos. In particular, whenever the Git
provider service is able to identify the license, the license_link field
is populated with the SPDX URL of the corresponding license.

Author: GlassOfWhiskey

.github/workflows/ci-build.yml

@@ -32,6 +32,14 @@ jobs:
           distribution: 'adopt'
           java-version: '17'
 
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.1'
+
+      - name: Install Licensee
+        run: gem install licensee
+
       - name: Install system packages
         run: |
           sudo apt-get -qq update

CONTRIBUTING.md

@@ -81,6 +81,9 @@ which allows them to be displayed in browsers as static prototypes.
 PostgreSQL is used to store information about `Workflow` and `QueuedWorkflow` 
 objects using [Spring Data JPA](https://docs.spring.io/spring-data/jpa/docs/current/reference/html/).
 
+The [Licensee](https://github.com/licensee/licensee) Ruby Gem is used to automatically infer license information from
+Git repositories.
+
 The application also uses a triple store to keep the RDF representing 
 workflows (gathered from [cwltool](https://github.com/common-workflow-language/cwltool)'s 
 `--print-rdf` functionality).

Dockerfile

@@ -1,3 +1,15 @@
+FROM maven:3-eclipse-temurin-17-alpine AS build-licensee
+
+RUN apk add --update \
+  alpine-sdk \
+  cmake \
+  heimdal-dev \
+  ruby-dev \
+  && rm -rf /var/cache/apk/*
+
+RUN gem install licensee
+
+
 FROM maven:3-eclipse-temurin-17-alpine
 MAINTAINER Stian Soiland-Reyes <[email protected]>
 
@@ -27,6 +39,8 @@ RUN apk add --update \
   libxml2-dev \
   libxml2-utils \
   libxslt-dev \
+  ruby \
+  heimdal \
   && rm -rf /var/cache/apk/*
 
 #wheel needed by ruamel.yaml for some reason
@@ -40,6 +54,9 @@ RUN mkdir /usr/share/maven/ref/repository
 RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 
+COPY --from=build-licensee /usr/lib/ruby/gems/   /usr/lib/ruby/gems/
+COPY --from=build-licensee /usr/bin/licensee    /usr/bin/
+
 # Top-level files (ignoring .git etc)
 ADD pom.xml LICENSE.md NOTICE.md README.md /usr/src/app/
 

README.md

@@ -113,6 +113,16 @@ system properties like `-Dspring.datasource.url=jdbc:postgresql://localhost:5432
 You will also need to have a SPARQL server such as [Apache Jena Fuseki](https://jena.apache.org/documentation/fuseki2/) running,
 by default on `localhost:3030`
 
+#### Ruby and Licensee
+
+To retrieve license information, CWL Viewe uses the [Licensee](https://github.com/licensee/licensee) Ruby Gem. To install it,
+[configure Ruby](https://www.ruby-lang.org/en/documentation/installation/) on your environment and then run
+
+```bash
+gem install licensee
+```
+
+
 ## Compiling and Running
 
 To compile you will need [Java 17](https://www.oracle.com/java/technologies/downloads/) or a compatible distribution

src/main/java/org/commonwl/view/cwl/CWLService.java

@@ -47,6 +47,7 @@
 import org.apache.jena.riot.RiotException;
 import org.commonwl.view.docker.DockerService;
 import org.commonwl.view.git.GitDetails;
+import org.commonwl.view.git.GitLicenseException;
 import org.commonwl.view.graphviz.ModelDotWriter;
 import org.commonwl.view.graphviz.RDFDotWriter;
 import org.commonwl.view.workflow.Workflow;
@@ -270,7 +271,7 @@ public Workflow parseWorkflowNative(Path workflowFile, String packedWorkflowId)
    * @return The constructed workflow object
    */
   public Workflow parseWorkflowWithCwltool(Workflow basicModel, Path workflowFile, Path workTree)
-      throws CWLValidationException {
+      throws CWLValidationException, GitLicenseException {
     GitDetails gitDetails = basicModel.getRetrievedFrom();
     String latestCommit = basicModel.getLastCommit();
     String packedWorkflowID = gitDetails.getPackedId();
@@ -452,15 +453,7 @@ public Workflow parseWorkflowWithCwltool(Workflow basicModel, Path workflowFile,
       licenseLink = licenseResult.next().get("license").toString();
     } else {
       // Check for "LICENSE"-like files in root of git repo
-      for (String licenseCandidate : new String[] {"LICENSE", "LICENSE.txt", "LICENSE.md"}) {
-        // FIXME: This might wrongly match lower-case "license.txt" in case-insensitive
-        // file systems
-        // but the URL would not work
-        if (Files.isRegularFile(workTree.resolve(licenseCandidate))) {
-          // Link to it by raw URL
-          licenseLink = basicModel.getRetrievedFrom().getRawUrl(null, licenseCandidate);
-        }
-      }
+      licenseLink = basicModel.getRetrievedFrom().getLicense(workTree);
     }
 
     // Docker link

src/main/java/org/commonwl/view/cwl/CWLToolRunner.java

@@ -24,6 +24,7 @@
 import java.util.Date;
 import org.apache.jena.query.QueryException;
 import org.commonwl.view.git.GitDetails;
+import org.commonwl.view.git.GitLicenseException;
 import org.commonwl.view.git.GitSemaphore;
 import org.commonwl.view.git.GitService;
 import org.commonwl.view.researchobject.ROBundleFactory;
@@ -111,7 +112,7 @@ public void createWorkflowFromQueued(QueuedWorkflow queuedWorkflow)
       queuedWorkflow.setCwltoolStatus(CWLToolStatus.ERROR);
       queuedWorkflow.setMessage("An error occurred when executing a query on the SPARQL store");
       FileUtils.deleteGitRepository(repo);
-    } catch (CWLValidationException ex) {
+    } catch (CWLValidationException | GitLicenseException ex) {
       String message = ex.getMessage();
       logger.error(
           "Workflow " + queuedWorkflow.getId() + " from " + gitInfo.toSummary() + " : " + message,

src/main/java/org/commonwl/view/git/GitDetails.java

@@ -20,17 +20,27 @@
 package org.commonwl.view.git;
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
 import java.io.Serializable;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.file.Path;
 import java.util.Objects;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /** Represents all the parameters necessary to access a file/directory with Git */
 @JsonIgnoreProperties(
     value = {"internalUrl"},
     ignoreUnknown = true)
 public class GitDetails implements Serializable {
 
+  private final Logger logger = LoggerFactory.getLogger(this.getClass());
+
+  private static final String SPDX_PREFIX = "https://spdx.org/licenses/";
+
   private String repoUrl;
   private String branch;
   private String path;
@@ -276,4 +286,53 @@ public String toSummary() {
     return String.format(
         "repoUrl: %s branch: %s path: %s packedId: %s", repoUrl, branch, path, packedId);
   }
+
+  /**
+   * Retrieves license details from the repo, if present.
+   *
+   * @param workTree the path to the locally cloned repo
+   * @return The license URI
+   */
+  public String getLicense(Path workTree) throws GitLicenseException {
+    try {
+      String[] command = {"licensee", "detect", "--json", workTree.toString()};
+      if (logger.isTraceEnabled()) {
+        logger.trace("Calling " + String.join(" ", command));
+      }
+      Process process = Runtime.getRuntime().exec(command, null);
+      ObjectMapper mapper = new ObjectMapper();
+      JsonNode jsonLicenses = mapper.readTree(process.getInputStream());
+      if (logger.isTraceEnabled()) {
+        logger.trace(
+            "Licensee retrieved the following licenses:\n" + jsonLicenses.toPrettyString());
+      }
+      int size = jsonLicenses.withArray("licenses").size();
+      if (size > 0) {
+        String licenseCandidate =
+            jsonLicenses.withArray("matched_files").get(0).get("filename").asText();
+        String licenseLink = getRawUrl(null, licenseCandidate);
+        if (logger.isWarnEnabled() && size > 1) {
+          logger.warn(
+              "There are "
+                  + size
+                  + " identified license files in the "
+                  + repoUrl
+                  + " repository. "
+                  + "Taking the first one: "
+                  + licenseLink);
+        }
+        String key = jsonLicenses.withArray("licenses").get(0).get("key").asText();
+        if (!"other".equals(key)) {
+          return SPDX_PREFIX + jsonLicenses.withArray("licenses").get(0).get("spdx_id").asText();
+        } else {
+          return licenseLink;
+        }
+      } else {
+        return null;
+      }
+    } catch (IOException e) {
+      throw new GitLicenseException(
+          "While attempting to detect license for " + workTree + ": " + e.getMessage(), e);
+    }
+  }
 }

src/main/java/org/commonwl/view/git/GitLicenseException.java

@@ -0,0 +1,18 @@
+package org.commonwl.view.git;
+
+import javax.validation.ValidationException;
+
+public class GitLicenseException extends ValidationException {
+
+  public GitLicenseException(String message) {
+    super(message);
+  }
+
+  public GitLicenseException(Throwable throwable) {
+    super(throwable);
+  }
+
+  public GitLicenseException(String message, Throwable throwable) {
+    super(message, throwable);
+  }
+}

src/test/java/org/commonwl/view/git/GitDetailsTest.java

@@ -21,8 +21,13 @@
 
 import static org.commonwl.view.git.GitDetails.normaliseUrl;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.Mockito.when;
 
+import java.io.File;
+import org.eclipse.jgit.lib.Repository;
 import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
 
 public class GitDetailsTest {
 
@@ -141,4 +146,28 @@ public void getNormaliseUrl() throws Exception {
     assertEquals("github.com/test/url/here", normaliseUrl("http://www.github.com/test/url/here"));
     assertEquals("github.com/test/url/here", normaliseUrl("http://github.com/test/url/here"));
   }
+
+  /** Retrieves license details from the repo, if present. */
+  @Test
+  public void getLicense() throws Exception {
+    Repository mockRepo = Mockito.mock(Repository.class);
+
+    when(mockRepo.getWorkTree()).thenReturn(new File("src/test/resources/cwl/licenses/apache/"));
+    assertEquals(
+        "https://spdx.org/licenses/Apache-2.0",
+        GENERIC_DETAILS.getLicense(mockRepo.getWorkTree().toPath()));
+
+    when(mockRepo.getWorkTree()).thenReturn(new File("src/test/resources/cwl/licenses/multiple/"));
+    assertEquals(
+        "https://spdx.org/licenses/Apache-2.0",
+        GENERIC_DETAILS.getLicense(mockRepo.getWorkTree().toPath()));
+
+    when(mockRepo.getWorkTree()).thenReturn(new File("src/test/resources/cwl/licenses/other/"));
+    assertEquals(
+        "https://could.com/be/anything/src/test/resources/cwl/licenses/other/LICENSE",
+        GENERIC_DETAILS.getLicense(mockRepo.getWorkTree().toPath()));
+
+    when(mockRepo.getWorkTree()).thenReturn(new File("src/test/resources/cwl/licenses/other/"));
+    assertNull(GENERIC_DETAILS.getLicense(mockRepo.getWorkTree().toPath()));
+  }
 }