Merge pull request #29 from blackducksoftware/task/romeara/28-failed-file-handling

GH-28 Update handling of files which failed to parse

Author: romeara

CHANGE_LOG.md

@@ -6,6 +6,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 ### Changed
+- (GH-28) Record files which failed to parse in a report, instead of failing the entire analysis
 - Updated `com.google.guava:guava` from `31.0-jre` to `31.1-jre`
 - Updated `org.ow2.asm:asm` from `9.2` to `9.3`
 - Updated `org.slf4j:slf4j-api` from `1.7.32` to `1.7.36`

docs/REPORT_FORMAT.md

@@ -54,6 +54,17 @@ The "method uses" report(s) are within the `referenced-method-uses` directory of
   - The qualified class and method name the reference is located in
 - methodUses[].uses[].lineNumber
   - Optional - the line number the reference is made on within the source file
+  
+## Broken Files Report
+
+The "broken files" report(s) are within the "broken-files" directory of the expanded report file. Each of these files is a JSON file, containing several elements:
+
+- brokenFiles[]
+  - Array of all broken files described by this report
+- brokenFiles[].path
+  - Path of the file which failed to parse
+- brokenFiles[].error
+  - Optional. Short text string describing the error which prevented parsing
 
 ## ID Generation
 

method-analyzer-core/src/main/java/com/synopsys/method/analyzer/core/MethodUseAnalyzer.java

@@ -27,7 +27,9 @@
 import java.net.InetAddress;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -39,6 +41,7 @@
 import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
 import com.google.common.collect.Multimap;
 import com.synopsys.method.analyzer.core.bytecode.ClassMethodReferenceVisitor;
 import com.synopsys.method.analyzer.core.model.MethodUse;
@@ -112,6 +115,7 @@ public Path analyze(Path sourceDirectory, Path outputDirectory, String outputFil
         Preconditions.checkArgument(Files.isDirectory(sourceDirectory), "The source path provided (%s) is not a directory", sourceDirectory.toString());
 
         Multimap<ReferencedMethod, MethodUse> references = null;
+        Map<Path, String> brokenFiles = new HashMap<>();
         ReportGenerator reportGenerator = new ReportGenerator(InetAddress.getLocalHost().getHostName(), sourceDirectory.toString(), codeLocationName);
 
         try (Stream<Path> files = Files.walk(sourceDirectory)) {
@@ -126,6 +130,14 @@ public Path analyze(Path sourceDirectory, Path outputDirectory, String outputFil
                 try (InputStream inputStream = Files.newInputStream(classFile)) {
                     ClassReader reader = new ClassReader(inputStream);
                     reader.accept(bytecodeAnalyzer, 0);
+                } catch (IllegalArgumentException e) {
+                    // IDETECT-3275: Instead of killing an entire analysis because of a single broken file, record the
+                    // broken file and move on
+                    if (Strings.nullToEmpty(e.getMessage()).startsWith("Unsupported class file major version")) {
+                        brokenFiles.put(classFile, Strings.nullToEmpty(e.getMessage()));
+                    } else {
+                        throw e;
+                    }
                 }
             }
 
@@ -136,7 +148,7 @@ public Path analyze(Path sourceDirectory, Path outputDirectory, String outputFil
                 .forEach(entry -> logger.debug("Found {} references to {}.{}({})",
                         entry.getValue().size(), entry.getKey().getMethodOwner(), entry.getKey().getMethodName(), entry.getKey().getInputs()));
 
-        return reportGenerator.generateReport(references, outputDirectory, outputFileName);
+        return reportGenerator.generateReport(references, brokenFiles, outputDirectory, outputFileName);
     }
 
 }

method-analyzer-core/src/main/java/com/synopsys/method/analyzer/core/report/BrokenFileJson.java

@@ -0,0 +1,88 @@
+/*
+ * method-use-analyzer
+ *
+ * Copyright (C) 2022 Synopsys Inc.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.synopsys.method.analyzer.core.report;
+
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * Represents JSON data for a file which failed to parse within an analyzed project
+ *
+ * <p>
+ * Fields within correspond to a defined file format - alterations require evaluation for backwards compatibility and
+ * required version control
+ *
+ * @author romeara
+ */
+public class BrokenFileJson {
+
+    private final String path;
+
+    private final String error;
+
+    public BrokenFileJson(String path, @Nullable String error) {
+        this.path = Objects.requireNonNull(path);
+        this.error = error;
+    }
+
+    public String getPath() {
+        return path;
+    }
+
+    @Nullable
+    public String getError() {
+        return error;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getPath(),
+                getError());
+    }
+
+    @Override
+    public boolean equals(@Nullable Object obj) {
+        boolean result = false;
+
+        if (obj instanceof BrokenFileJson) {
+            BrokenFileJson compare = (BrokenFileJson) obj;
+
+            result = Objects.equals(compare.getPath(), getPath())
+                    && Objects.equals(compare.getError(), getError());
+        }
+
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass()).omitNullValues()
+                .add("path", getPath())
+                .add("error", getError())
+                .toString();
+    }
+
+}

method-analyzer-core/src/main/java/com/synopsys/method/analyzer/core/report/BrokenFilesReportJson.java

@@ -0,0 +1,78 @@
+/*
+ * method-use-analyzer
+ *
+ * Copyright (C) 2022 Synopsys Inc.
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.synopsys.method.analyzer.core.report;
+
+import java.util.List;
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * Represents full JSON contents for a report on broken files which failed to parse within an analyzed project
+ *
+ * <p>
+ * Fields within correspond to a defined file format - alterations require evaluation for backwards compatibility and
+ * required version control
+ *
+ * @author romeara
+ */
+public final class BrokenFilesReportJson {
+
+    private final List<BrokenFileJson> brokenFiles;
+
+    public BrokenFilesReportJson(List<BrokenFileJson> brokenFiles) {
+        this.brokenFiles = Objects.requireNonNull(brokenFiles);
+    }
+
+    public List<BrokenFileJson> getBrokenFiles() {
+        return brokenFiles;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getBrokenFiles());
+    }
+
+    @Override
+    public boolean equals(@Nullable Object obj) {
+        boolean result = false;
+
+        if (obj instanceof BrokenFilesReportJson) {
+            BrokenFilesReportJson compare = (BrokenFilesReportJson) obj;
+
+            result = Objects.equals(compare.getBrokenFiles(), getBrokenFiles());
+        }
+
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass()).omitNullValues()
+                .add("brokenFiles", getBrokenFiles())
+                .toString();
+    }
+
+}

method-analyzer-core/src/main/java/com/synopsys/method/analyzer/core/report/ReportGenerator.java

@@ -31,13 +31,15 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Objects;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
@@ -67,12 +69,16 @@ public class ReportGenerator {
 
     private static final String REFERENCE_METHOD_USE_LABEL = "referenced-method-uses";
 
+    private static final String BROKEN_FILE_LABEL = "broken-files";
+
     // TODO romeara - In the future, should we investigate dynamic call-through to make this an actual handshake, and
     // not a manual constant? Aka ask the service what is allowed?
     // This is determined in conjunction with what the KB cloud services will allow - DO NOT adjust this number without
     // consultation with the maximum allowed by that service
     private static final int REFERENCE_MAX_CHUNK_SIZE = 1000;
 
+    private static final int BROKEN_FILE_MAX_CHUNK_SIZE = 1000;
+
     private static final Gson GSON = new Gson();
 
     /** Logger reference to output information to the application log files */
@@ -104,9 +110,33 @@ public ReportGenerator(String hostName, String analyzedDirectory, @Nullable Stri
      * @return The path of the generated report on the file system
      * @throws IOException
      *             If there is an error writing the report to the file system
+     * @deprecated Use {@link #generateReport(Multimap, Map, Path, String)} instead
      */
+    @Deprecated
     public Path generateReport(Multimap<ReferencedMethod, MethodUse> references, Path outputDirectory, String outputFileName) throws IOException {
+        return generateReport(references, Collections.emptyMap(), outputDirectory, outputFileName);
+    }
+
+    /**
+     * Generates a report of the provided method references within a project
+     *
+     * @param references
+     *            The references to describe in a report
+     * @param brokenFiles
+     *            Paths of any files for which analysis was attempted, by failed due to parser incompatibly of broken
+     *            file formatting (mapped to a message indicating the specific error)
+     * @param outputDirectory
+     *            The directory to output the report to
+     * @param outputFileName
+     *            The file name to use for the report (without extension)
+     * @return The path of the generated report on the file system
+     * @throws IOException
+     *             If there is an error writing the report to the file system
+     */
+    public Path generateReport(Multimap<ReferencedMethod, MethodUse> references, Map<Path, String> brokenFiles, Path outputDirectory, String outputFileName)
+            throws IOException {
         Objects.requireNonNull(references);
+        Objects.requireNonNull(brokenFiles);
         Objects.requireNonNull(outputDirectory);
         Objects.requireNonNull(outputFileName);
 
@@ -125,7 +155,11 @@ public Path generateReport(Multimap<ReferencedMethod, MethodUse> references, Pat
             methodUses.add(new ReferencedMethodUsesJson(id.getSignature(), entry.getKey(), entry.getValue()));
         }
 
-        return writeReport(destinationFile, uniqueMethodKeys, methodUses);
+        List<BrokenFileJson> brokenFileRecords = brokenFiles.entrySet().stream()
+                .map(entry -> new BrokenFileJson(entry.getKey().toString(), entry.getValue()))
+                .collect(Collectors.toList());
+
+        return writeReport(destinationFile, uniqueMethodKeys, methodUses, brokenFileRecords);
     }
 
     /**
@@ -137,14 +171,18 @@ public Path generateReport(Multimap<ReferencedMethod, MethodUse> references, Pat
      *            List of unique, opaque keys referencing discovered methods
      * @param methodUses
      *            Detailed description(s) of the method uses discovered
+     * @param brokenFiles
+     *            Description of files which failed to parse
      * @return The path the report was written to
      * @throws IOException
      *             If there is an error writing the report
      */
-    private Path writeReport(Path destinationFile, List<MethodIdJson> uniqueMethodKeys, List<ReferencedMethodUsesJson> methodUses) throws IOException {
+    private Path writeReport(Path destinationFile, List<MethodIdJson> uniqueMethodKeys, List<ReferencedMethodUsesJson> methodUses,
+            List<BrokenFileJson> brokenFiles) throws IOException {
         Objects.requireNonNull(destinationFile);
         Objects.requireNonNull(uniqueMethodKeys);
         Objects.requireNonNull(methodUses);
+        Objects.requireNonNull(brokenFiles);
 
         Map<Path, Path> outputFileMapping = new HashMap<>();
 
@@ -185,6 +223,18 @@ private Path writeReport(Path destinationFile, List<MethodIdJson> uniqueMethodKe
             }
         }
 
+        List<List<BrokenFileJson>> brokenFilePartitions = Lists.partition(brokenFiles, BROKEN_FILE_MAX_CHUNK_SIZE);
+
+        for (int index = 0; index < brokenFilePartitions.size(); index++) {
+            List<BrokenFileJson> brokenFilePartition = brokenFilePartitions.get(index);
+
+            Path brokenFilesFile = workingDirectory.resolve(BROKEN_FILE_LABEL + "-" + index + ".json");
+            try (BufferedWriter writer = Files.newBufferedWriter(brokenFilesFile)) {
+                GSON.toJson(new BrokenFilesReportJson(brokenFilePartition), writer);
+                outputFileMapping.put(brokenFilesFile, Paths.get(BROKEN_FILE_LABEL).resolve(brokenFilesFile.getFileName()));
+            }
+        }
+
         writeZipFile(destinationFile, outputFileMapping);
 
         // GH-22: Explicitly clean up temporary files once use of them is complete