Add a graph-based Java module with syntax normalization and dead code detection. #2766

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

LundiNord wants to merge 162 commits into jplag:develop from LundiNord:main

.gitignore

-Original file line number
+Diff line change
@@ Expand Up / @@ -2,6 +2,8 @@ target/ @@
     .project
     .settings/
     .classpath
+    .factorypath
+    website/node_modules
     *.class
@@ Expand Down @@

cli/pom.xml

-Original file line number
+Diff line change
@@ Expand Up / @@ -30,6 +30,11 @@ @@
                 <groupId>de.jplag</groupId>
                 <artifactId>java</artifactId>
             </dependency>
+            <dependency>
+                <groupId>de.jplag</groupId>
+                <artifactId>java-cpg</artifactId>
+                <version>${revision}</version>
+            </dependency>
             <dependency>
                 <groupId>de.jplag</groupId>
                 <artifactId>python-3</artifactId>
@@ Expand Down @@

core/src/main/java/de/jplag/Submission.java

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -1,25 +1,12 @@
  
    package de.jplag;

    import static de.jplag.SubmissionState.CANNOT_PARSE;

    import static de.jplag.SubmissionState.NOTHING_TO_PARSE;

    import static de.jplag.SubmissionState.TOO_SMALL;

    import static de.jplag.SubmissionState.UNPARSED;

    import static de.jplag.SubmissionState.VALID;

    import static de.jplag.SubmissionState.*;

    import java.io.File;

    import java.io.IOException;

    import java.nio.file.Files;

    import java.nio.file.Path;

    import java.util.ArrayList;

    import java.util.Collection;

    import java.util.Collections;

    import java.util.HashMap;

    import java.util.HashSet;

    import java.util.List;

    import java.util.Map;

    import java.util.Objects;

    import java.util.Optional;

    import java.util.Set;

    import java.util.*;

    import org.slf4j.Logger;

    import org.slf4j.LoggerFactory;

    @@ -71,11 +58,24 @@ public Submission(String name, File submissionRootFile, boolean isNew, Collectio
  
            state = UNPARSED;

        }

        private static File createErrorDirectory(String... subdirectoryNames) {

            File subdirectory = Path.of(JPlagOptions.ERROR_FOLDER, subdirectoryNames).toFile();

            if (!subdirectory.exists()) {

                subdirectory.mkdirs();

            }

            return subdirectory;

        }

        @Override

        public int compareTo(Submission other) {

            return name.compareTo(other.name);

        }

        @Override

        public int hashCode() {

            return Objects.hash(name);

        }

        @Override

        public boolean equals(Object obj) {

            if (obj == this) {

    @@ -88,8 +88,8 @@ public boolean equals(Object obj) {
  
        }

        @Override

        public int hashCode() {

            return Objects.hash(name);

        public String toString() {

            return name;

        }

        /**

    @@ -100,6 +100,14 @@ public JPlagComparison getBaseCodeComparison() {
  
            return baseCodeComparison;

        }

        /**

         * Sets the base code comparison.

         * @param baseCodeComparison is submissions matches with the base code.

         */

        public void setBaseCodeComparison(JPlagComparison baseCodeComparison) {

            this.baseCodeComparison = baseCodeComparison;

        }

        /**

         * Provided all source code files.

         * @return a collection of files this submission consists of.

    @@ -153,6 +161,14 @@ public List<Token> getTokenList() {
  
            return tokenList;

        }

        /**

         * Sets the tokens that have been parsed from the files this submission consists of.

         * @param tokenList is the list of these tokens.

         */

        public void setTokenList(List<Token> tokenList) {

            this.tokenList = Collections.unmodifiableList(new ArrayList<>(tokenList));

        }

        /**

         * @return true if a comparison between the submission and the base code is available. Does not imply if there are

         * matches to the base code.

    @@ -185,22 +201,6 @@ public boolean isNew() {
  
            return isNew;

        }

        /**

         * Sets the base code comparison.

         * @param baseCodeComparison is submissions matches with the base code.

         */

        public void setBaseCodeComparison(JPlagComparison baseCodeComparison) {

            this.baseCodeComparison = baseCodeComparison;

        }

        /**

         * Sets the tokens that have been parsed from the files this submission consists of.

         * @param tokenList is the list of these tokens.

         */

        public void setTokenList(List<Token> tokenList) {

            this.tokenList = Collections.unmodifiableList(new ArrayList<>(tokenList));

        }

        /**

         * String representation of the code files contained in this submission, annotated with all tokens.

         * @return the annotated code as string.

    @@ -209,11 +209,6 @@ public String getTokenAnnotatedSourceCode() {
  
            return TokenPrinter.printTokens(tokenList, submissionRootFile);

        }

        @Override

        public String toString() {

            return name;

        }

        /**

         * This method is used to copy files that can not be parsed to a special folder.

         */

    @@ -229,14 +224,6 @@ private void copySubmission() {
  
            }

        }

        private static File createErrorDirectory(String... subdirectoryNames) {

            File subdirectory = Path.of(JPlagOptions.ERROR_FOLDER, subdirectoryNames).toFile();

            if (!subdirectory.exists()) {

                subdirectory.mkdirs();

            }

            return subdirectory;

        }

        /**

         * Parse files of the submission.

         * @param debugParser specifies if the submission should be copied upon parsing errors.

    @@ -348,7 +335,9 @@ public Map<File, Integer> getTokenCountPerFile() {
  
                    fileTokenCount.put(file, 0);

                }

                for (Token token : this.tokenList) {

                    fileTokenCount.put(token.getFile(), fileTokenCount.get(token.getFile()) + 1);

                    File tokenfile = token.getFile();

                    fileTokenCount.computeIfAbsent(tokenfile, k -> 0);

                    fileTokenCount.put(tokenfile, fileTokenCount.get(tokenfile) + 1);

                }

            }

            return fileTokenCount;

coverage-report/pom.xml

-Original file line number
+Diff line change
@@ Expand Up / @@ -43,6 +43,10 @@ @@
                 <groupId>de.jplag</groupId>
                 <artifactId>java</artifactId>
             </dependency>
+            <dependency>
+                <groupId>de.jplag</groupId>
+                <artifactId>java-cpg</artifactId>
+            </dependency>
             <dependency>
                 <groupId>de.jplag</groupId>
                 <artifactId>python-3</artifactId>
@@ Expand Down @@

endtoend-testing/src/test/java/de/jplag/endtoend/architecture/PomVersionTest.java

-Original file line number
+Diff line change
@@ Expand Up / @@ -17,6 +17,7 @@ @@
     import org.jdom2.xpath.XPathBuilder;
     import org.jdom2.xpath.XPathExpression;
     import org.jdom2.xpath.XPathFactory;
+    import org.jetbrains.annotations.NotNull;
     import org.junit.jupiter.api.Assertions;
     import org.junit.jupiter.api.DisplayName;
     import org.junit.jupiter.api.Test;
@@ Expand Down Expand Up / @@ -50,6 +51,34 @@ class PomVersionTest { @@
         private static final File projectRoot = new File("..");
         private static final File projectRootPom = new File(projectRoot, POM_XML_NAME);
+        private static final List<String> ALLOWED_EXTERNAL_DEPENDENCIES = List.of("de.fraunhofer.aisec:cpg-core", "de.fraunhofer.aisec:cpg-language-java",
+                "de.fraunhofer.aisec:cpg-analysis", "org.jetbrains.kotlin:kotlin-stdlib-jdk8", "org.jetbrains.kotlin:kotlin-test");
+        private static final List<String> ALLOWED_EXTERNAL_PLUGINS = List.of("org.jetbrains.kotlin:kotlin-maven-plugin");
+        static List<File> getAllPomFiles() {
+            List<File> collector = new ArrayList<>();
+            getAllPomFiles(projectRoot, collector);
+            return collector;
+        }
+        static List<File> getAllChildPomFiles() {
+            List<File> poms = getAllPomFiles();
+            poms.remove(projectRootPom);
+            return poms;
+        }
+        static void getAllPomFiles(File scanDir, List<File> collector) {
+            for (File file : scanDir.listFiles()) {
+                if (file.isFile() && file.getName().equals(POM_XML_NAME)) {
+                    collector.add(file);
+                }
+                if (file.isDirectory()) {
+                    getAllPomFiles(file, collector);
+                }
+            }
+        }
         @ParameterizedTest
         @MethodSource("getAllChildPomFiles")
         @DisplayName("Ensure all child poms use " + REVISION_REFERENCE + " to reference parent")
@@ Expand Down Expand Up @@
             return expression.evaluate(doc);
         }
-        private void failForVersionTag(Element versionTag) {
+        private void failForVersionTag(@NotNull Element versionTag) {
             Namespace ns = versionTag.getNamespace();
             Element definition = versionTag.getParentElement();
-            String definitionString = "{groupId: " + definition.getChildText("groupId", ns) + ", artifactId:" + definition.getChildText("artifactId", ns)
-                    + ", version:" + versionTag.getText() + "}";
+            String groupId = definition.getChildText("groupId", ns);
+            String artifactId = definition.getChildText("artifactId", ns);
+            // Handle inherited groupId from parent
+            if (groupId == null) {
+                groupId = "de.jplag";
+            }
+            String key = groupId + ":" + artifactId;
+            // Allow external dependencies and plugins
+            if (ALLOWED_EXTERNAL_DEPENDENCIES.contains(key) || ALLOWED_EXTERNAL_PLUGINS.contains(key)) {
+                return;
+            }
+            // Allow property-based external dependencies
+            if (groupId.startsWith("${") && (ALLOWED_EXTERNAL_DEPENDENCIES.stream().anyMatch(dep -> dep.endsWith(":" + artifactId))
+                    || ALLOWED_EXTERNAL_PLUGINS.stream().anyMatch(plugin -> plugin.endsWith(":" + artifactId)))) {
+                return;
+            }
+            // Allow internal JPlag dependencies with ${revision}
+            if ("de.jplag".equals(groupId) && "${revision}".equals(versionTag.getText())) {
+                return;
+            }
+            String definitionString = "{groupId: " + groupId + ", artifactId:" + artifactId + ", version:" + versionTag.getText() + "}";
             Assertions.fail("Invalid version tag found for: " + definitionString);
         }
-        static List<File> getAllPomFiles() {
-            List<File> collector = new ArrayList<>();
-            getAllPomFiles(projectRoot, collector);
-            return collector;
-        }
-        static List<File> getAllChildPomFiles() {
-            List<File> poms = getAllPomFiles();
-            poms.remove(projectRootPom);
-            return poms;
-        }
-        static void getAllPomFiles(File scanDir, List<File> collector) {
-            for (File file : scanDir.listFiles()) {
-                if (file.isFile() && file.getName().equals(POM_XML_NAME)) {
-                    collector.add(file);
-                }
-                if (file.isDirectory()) {
-                    getAllPomFiles(file, collector);
-                }
-            }
-        }
     }

languages/java-cpg/pom.xml

-Original file line number
+Diff line change
@@ -0,0 +1,111 @@
+    <?xml version="1.0" encoding="UTF-8"?>
+    <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+        <modelVersion>4.0.0</modelVersion>
+        <parent>
+            <groupId>de.jplag</groupId>
+            <artifactId>languages</artifactId>
+            <version>${revision}</version>
+        </parent>
+        <artifactId>java-cpg</artifactId>
+        <properties>
+            <cpg.version>8.3.0</cpg.version>
+            <cpg.groupId>de.fraunhofer.aisec</cpg.groupId>
+            <kotlin.version>2.3.0</kotlin.version>
+        </properties>
+        <dependencies>
+            <dependency>
+                <groupId>${cpg.groupId}</groupId>
+                <artifactId>cpg-core</artifactId>
+                <version>${cpg.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>org.apache.logging.log4j</groupId>
+                        <artifactId>log4j-slf4j2-impl</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+            <dependency>
+                <groupId>${cpg.groupId}</groupId>
+                <artifactId>cpg-language-java</artifactId>
+                <version>${cpg.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>${cpg.groupId}</groupId>
+                <artifactId>cpg-analysis</artifactId>
+                <version>${cpg.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.jetbrains.kotlin</groupId>
+                <artifactId>kotlin-stdlib-jdk8</artifactId>
+                <version>${kotlin.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.jetbrains.kotlin</groupId>
+                <artifactId>kotlin-test</artifactId>
+                <version>${kotlin.version}</version>
+                <scope>test</scope>
+            </dependency>
+            <dependency>
+                <groupId>de.jplag</groupId>
+                <artifactId>jplag</artifactId>
+                <scope>test</scope>
+            </dependency>
+            <dependency>
+                <groupId>de.jplag</groupId>
+                <artifactId>java</artifactId>
+                <scope>test</scope>
+            </dependency>
+        </dependencies>
+        <build>
+            <plugins>
+                <plugin>
+                    <groupId>org.jetbrains.kotlin</groupId>
+                    <artifactId>kotlin-maven-plugin</artifactId>
+                    <version>${kotlin.version}</version>
+                    <configuration>
+                        <jvmTarget>1.8</jvmTarget>
+                        <args>
+                            <arg>-Xcontext-receivers</arg>
+                        </args>
+                    </configuration>
+                    <executions>
+                        <execution>
+                            <id>compile</id>
+                            <goals>
+                                <goal>compile</goal>
+                            </goals>
+                            <phase>process-sources</phase>
+                        </execution>
+                        <execution>
+                            <id>test-compile</id>
+                            <goals>
+                                <goal>test-compile</goal>
+                            </goals>
+                            <phase>test-compile</phase>
+                        </execution>
+                    </executions>
+                </plugin>
+                <plugin>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-javadoc-plugin</artifactId>
+                    <configuration>
+                        <sourcepath>src/main/java</sourcepath>
+                        <sourceFileIncludes>
+                            <include>**/*.java</include>
+                        </sourceFileIncludes>
+                        <additionalOptions>
+                            <additionalOption>-Xdoclint:none</additionalOption>
+                        </additionalOptions>
+                        <failOnError>false</failOnError>
+                    </configuration>
+                </plugin>
+            </plugins>
+        </build>
+    </project>

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add a graph-based Java module with syntax normalization and dead code detection. #2766

Uh oh!

Diff view

Diff view

Uh oh!

There are no files selected for viewing

Uh oh!

Add a graph-based Java module with syntax normalization and dead code detection. #2766

Are you sure you want to change the base?

Uh oh!

Add a graph-based Java module with syntax normalization and dead code detection. #2766

Uh oh!

Diff view

Diff view

Uh oh!

There are no files selected for viewing

Uh oh!