Merge pull request #176 from VariantSync/diff-interface

Unified differ interface and source tracing improvements

Author: pmbittner

README.md

@@ -10,7 +10,7 @@
 
 DiffDetective is an open-source Java library for variability-aware source code differencing and the **analysis of version histories of software product lines**. This means that DiffDetective can **turn a generic differencer into a variability-aware differencer** by means of a pre- or post-processing. DiffDetective is centered around **formally verified** data structures for variability (variation trees) and variability-aware diffs (variation diffs). These data structures are **generic**, and DiffDetective currently implements **C preprocessor support** to parse respective annotations when used to implement variability. The picture below depicts the process of variability-aware differencing.
 
-<img alt="Variability-Aware Differencing Overview" src="docs/variability-aware-differencing.png" height="500" />
+<img alt="Variability-Aware Differencing Overview" src="src/main/java/org/variantsync/diffdetective/variation/diff/doc-files/variability-aware-differencing.png" height="500" />
 
 Given two states of a C-preprocessor annotated source code file (left), for example before and after a commit, DiffDetective constructs a variability-aware diff (right) that distinguishes changes to source code from changes to variability annotations. DiffDetective can construct such a variation diff either, by first using a generic differencer, and separating the information (center path), or by first parsing both input versions to an abstract representation, a variation tree (center top and bottom), and constructing a variation diff using a tree differencing algorithm in a second step.
 

src/main/java/org/variantsync/diffdetective/diff/git/GitDiffer.java

@@ -21,9 +21,11 @@
 import org.variantsync.diffdetective.variation.DiffLinesLabel;
 import org.variantsync.diffdetective.variation.diff.VariationDiff;
 import org.variantsync.diffdetective.variation.diff.parse.VariationDiffParser;
+import org.variantsync.diffdetective.variation.tree.source.GitSource;
 
 import java.io.*;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
@@ -252,6 +254,7 @@ private static CommitDiffResult getPatchDiffs(
 
                     final VariationDiff<DiffLinesLabel> variationDiff = VariationDiffParser.createVariationDiff(
                             fullDiff,
+                            new GitSource(repository, childCommit.getId().name(), Path.of(filename)),
                             repository.getParseOptions().variationDiffParseOptions()
                     );
 

src/main/java/org/variantsync/diffdetective/diff/git/GitPatch.java

@@ -1,17 +1,19 @@
 package org.variantsync.diffdetective.diff.git;
 
+import java.util.List;
+
 import org.eclipse.jgit.diff.DiffEntry;
 import org.variantsync.diffdetective.diff.text.TextBasedDiff;
+import org.variantsync.diffdetective.util.Source;
 import org.variantsync.diffdetective.variation.diff.Time;
 import org.variantsync.diffdetective.variation.diff.VariationDiff; // For Javadoc
-import org.variantsync.diffdetective.variation.diff.source.VariationDiffSource;
 
 /**
  * Interface for patches from a git repository.
  * A git patch is a {@link TextBasedDiff} from which {@link VariationDiff}s can be created.
  *
  */
-public interface GitPatch extends VariationDiffSource, TextBasedDiff {
+public interface GitPatch extends Source, TextBasedDiff {
     /**
      * Minimal default implementation of {@link GitPatch}
      * @param getDiff The diff in text form.
@@ -41,6 +43,16 @@ public GitPatch shallowClone() {
         public String toString() {
             return oldFileName + "@ " + getParentCommitHash + " (parent) to " + newFileName + " @ " + getCommitHash + " (child)";
         }
+
+        @Override
+        public String getSourceExplanation() {
+            return "SimpleGitPatch";
+        }
+
+        @Override
+        public List<Object> getSourceArguments() {
+            return List.of(getChangeType(), oldFileName(), newFileName(), getCommitHash(), getParentCommitHash());
+        }
     }
 
     /**

src/main/java/org/variantsync/diffdetective/diff/git/PatchDiff.java

@@ -123,4 +123,9 @@ public String toString() {
     public GitPatch shallowClone() {
         return new GitPatch.SimpleGitPatch(getDiff(), getChangeType(), getFileName(Time.BEFORE), getFileName(Time.AFTER), getCommitHash(), getParentCommitHash());
     }
+
+    @Override
+    public String getSourceExplanation() {
+        return "PatchDiff";
+    }
 }

src/main/java/org/variantsync/diffdetective/examplesearch/ExampleFinder.java

@@ -11,6 +11,7 @@
 import org.variantsync.diffdetective.show.Show;
 import org.variantsync.diffdetective.util.Assert;
 import org.variantsync.diffdetective.util.IO;
+import org.variantsync.diffdetective.util.Source;
 import org.variantsync.diffdetective.util.StringUtils;
 import org.variantsync.diffdetective.variation.DiffLinesLabel;
 import org.variantsync.diffdetective.variation.diff.Time;
@@ -24,7 +25,6 @@
 import org.variantsync.diffdetective.variation.diff.serialize.edgeformat.DefaultEdgeLabelFormat;
 import org.variantsync.diffdetective.variation.diff.serialize.nodeformat.MappingsDiffNodeFormat;
 import org.variantsync.diffdetective.variation.diff.serialize.treeformat.CommitDiffVariationDiffLabelFormat;
-import org.variantsync.diffdetective.variation.diff.source.VariationDiffSource;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -106,14 +106,10 @@ private boolean checkIfExample(Analysis analysis, String localDiff) {
         // We do not want a variationDiff for the entire file but only for the local change to have a small example.
         final VariationDiff<DiffLinesLabel> localTree;
         try {
-            localTree = VariationDiff.fromDiff(localDiff, new VariationDiffParseOptions(annotationParser, true, true));
+            localTree = VariationDiff.fromDiff(localDiff, Source.findFirst(variationDiff, GitPatch.class), new VariationDiffParseOptions(annotationParser, true, true));
             // Not every local diff can be parsed to a VariationDiff because diffs are unaware of the underlying language (i.e., CPP).
             // We want only running examples whose diffs describe entire diff trees for easier understanding.
-            if (isGoodExample.test(localTree)) {
-                Assert.assertTrue(variationDiff.getSource() instanceof GitPatch);
-                final GitPatch variationDiffSource = (GitPatch) variationDiff.getSource();
-                localTree.setSource(variationDiffSource.shallowClone());
-            } else {
+            if (!isGoodExample.test(localTree)) {
                 return false;
             }
         } catch (DiffParseException e) {
@@ -149,9 +145,9 @@ public boolean analyzeVariationDiff(Analysis analysis) {
     }
 
     private void exportExample(final Analysis analysis, final String tdiff, final VariationDiff<DiffLinesLabel> vdiff, Path outputDir) {
-        Assert.assertTrue(vdiff.getSource() instanceof GitPatch);
         final Repository repo = analysis.getRepository();
-        final GitPatch patch = (GitPatch) vdiff.getSource();
+        final GitPatch patch = Source.findFirst(vdiff, GitPatch.class);
+        Assert.assertNotNull(patch);
         outputDir = outputDir.resolve(Path.of(repo.getRepositoryName() + "_" + patch.getCommitHash()));
         final String filename = patch.getFileName(Time.AFTER);
 
@@ -185,8 +181,8 @@ private void exportExample(final Analysis analysis, final String tdiff, final Va
     }
 
     static String getDiff(final VariationDiff<?> tree) {
-        final VariationDiffSource source = tree.getSource();
-        Assert.assertTrue(source instanceof TextBasedDiff);
-        return ((TextBasedDiff) source).getDiff();
+        TextBasedDiff textBasedDiff = Source.findFirst(tree, TextBasedDiff.class);
+        Assert.assertNotNull(textBasedDiff);
+        return textBasedDiff.getDiff();
     }
 }

src/main/java/org/variantsync/diffdetective/experiments/thesis_bm/ConstructionValidation.java

@@ -5,6 +5,7 @@
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
+import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
@@ -42,6 +43,7 @@
 import org.variantsync.diffdetective.variation.diff.Time;
 import org.variantsync.diffdetective.variation.diff.filter.VariationDiffFilter;
 import org.variantsync.diffdetective.variation.diff.parse.VariationDiffParser;
+import org.variantsync.diffdetective.variation.tree.source.GitSource;
 import org.variantsync.functjonal.category.InplaceSemigroup;
 import org.variantsync.functjonal.map.MergeMap;
 
@@ -375,6 +377,7 @@ private void counts(VariationDiff<DiffLinesLabel> tree, VariationDiffStatistics
     }
 
     private VariationDiff<DiffLinesLabel> parseVariationTree(Analysis analysis, RevCommit commit) throws IOException, DiffParseException {
+        String fileName = analysis.getCurrentPatch().getFileName(AFTER);
         try (BufferedReader afterFile =
             new BufferedReader(
                 /*
@@ -386,10 +389,14 @@ private VariationDiff<DiffLinesLabel> parseVariationTree(Analysis analysis, RevC
                     GitDiffer.getBeforeFullFile(
                         analysis.getRepository(),
                         commit,
-                        analysis.getCurrentPatch().getFileName(AFTER)),
+                        fileName),
                     0xfeff)) // BOM, same as GitDiffer.BOM_PATTERN
         ) {
-            return VariationDiffParser.createVariationTree(afterFile, analysis.getRepository().getParseOptions().variationDiffParseOptions());
+            return VariationDiffParser.createVariationTree(
+                afterFile,
+                new GitSource(analysis.getRepository(), commit.getId().name(), Path.of(fileName)),
+                analysis.getRepository().getParseOptions().variationDiffParseOptions()
+            );
         }
     }
 

src/main/java/org/variantsync/diffdetective/experiments/thesis_es/UnparseAnalysis.java

@@ -9,6 +9,7 @@
 import org.variantsync.diffdetective.util.CSV;
 import org.variantsync.diffdetective.util.FileUtils;
 import org.variantsync.diffdetective.util.IO;
+import org.variantsync.diffdetective.util.Source;
 import org.variantsync.diffdetective.util.StringUtils;
 import org.variantsync.diffdetective.variation.DiffLinesLabel;
 import org.variantsync.diffdetective.variation.VariationUnparser;
@@ -17,7 +18,6 @@
 import org.variantsync.diffdetective.variation.diff.construction.JGitDiff;
 import org.variantsync.diffdetective.variation.diff.parse.VariationDiffParseOptions;
 import org.variantsync.diffdetective.variation.tree.VariationTree;
-import org.variantsync.diffdetective.variation.tree.source.VariationTreeSource;
 
 public class UnparseAnalysis implements Analysis.Hooks {
 
@@ -215,7 +215,7 @@ public static String removeWhitespace(String string, boolean diff) {
     public static String parseUnparseTree(String text, VariationDiffParseOptions option) {
         String temp = "b";
         try {
-            VariationTree<DiffLinesLabel> tree = VariationTree.fromText(text, VariationTreeSource.Unknown, option);
+            VariationTree<DiffLinesLabel> tree = VariationTree.fromText(text, Source.Unknown, option);
             temp = VariationUnparser.unparseTree(tree);
         } catch (Exception e) {
             e.printStackTrace();
@@ -226,7 +226,7 @@ public static String parseUnparseTree(String text, VariationDiffParseOptions opt
     public static String parseUnparseDiff(String textDiff, VariationDiffParseOptions option) {
         String temp = "b";
         try {
-            VariationDiff<DiffLinesLabel> diff = VariationDiff.fromDiff(textDiff, option);
+            VariationDiff<DiffLinesLabel> diff = VariationDiff.fromDiff(textDiff, Source.Unknown, option);
             temp = VariationUnparser.unparseDiff(diff);
         } catch (Exception e) {
             e.printStackTrace();

src/main/java/org/variantsync/diffdetective/experiments/views/result/ViewEvaluation.java

@@ -85,8 +85,8 @@ public String toCSV(String delimiter) {
 //                repo.getRepositoryName(),
                 commit,
                 file,
-                relevance.getFunctionName(),
-//                getQueryArguments(),
+                relevance.getSourceExplanation(),
+//                relevance.getSourceArguments(),
                 msNaive,
                 msOptimized,
                 diffStatistics.nodeCount,

src/main/java/org/variantsync/diffdetective/load/GitLoader.java

@@ -1,7 +1,6 @@
 package org.variantsync.diffdetective.load;
 
 import net.lingala.zip4j.ZipFile;
-import net.lingala.zip4j.exception.ZipException;
 import org.apache.commons.io.FilenameUtils;
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.api.errors.GitAPIException;
@@ -86,10 +85,9 @@ public static Git fromZip(Path pathToZip) {
             return fromDirectory(unzippedRepoName);
         }
 
-        try {
-            ZipFile zipFile = new ZipFile(pathToZip.toFile());
+        try (ZipFile zipFile = new ZipFile(pathToZip.toFile())) {
             zipFile.extractAll(targetDir.toString());
-        } catch (ZipException e) {
+        } catch (IOException e) {
             Logger.warn("Failed to extract git repo from {} to {}", pathToZip, targetDir);
             return null;
         }

src/main/java/org/variantsync/diffdetective/mining/RWCompositePatternTreeFormat.java

@@ -1,22 +1,25 @@
 package org.variantsync.diffdetective.mining;
 
-import org.variantsync.diffdetective.variation.diff.source.VariationDiffSource;
+import java.util.List;
+
+import org.variantsync.diffdetective.util.FileSource;
+import org.variantsync.diffdetective.util.Source;
 import org.variantsync.diffdetective.variation.diff.serialize.treeformat.VariationDiffLabelFormat;
-import org.variantsync.diffdetective.variation.diff.source.PatchFile;
 
 public class RWCompositePatternTreeFormat implements VariationDiffLabelFormat {
     @Override
-    public VariationDiffSource fromLabel(String label) {
+    public Source fromLabel(String label) {
         throw new UnsupportedOperationException("Cannot read");
     }
 
     @Override
-    public String toLabel(VariationDiffSource variationDiffSource) {
-        if (variationDiffSource instanceof PatchFile p) {
-            final String fileName = p.path().getFileName().toString();
-            return fileName.substring(0, fileName.indexOf('.')).replaceAll("_", " ");
+    public String toLabel(Source variationDiffSource) {
+        List<FileSource> pathSources = Source.findAll(variationDiffSource, FileSource.class);
+        if (pathSources.size() != 1) {
+            throw new IllegalArgumentException("Expected a single path source but got:\n" + Source.fullExplanation(variationDiffSource));
         }
 
-        throw new IllegalArgumentException("Expected a PatchFile but got " + variationDiffSource);
+        final String fileName = pathSources.get(0).getPath().getFileName().toString();
+        return fileName.substring(0, fileName.indexOf('.')).replaceAll("_", " ");
     }
 }