feat: support ignore paths (#214) (#220)

* feat: support ignore paths (#214)

* address comments

* config should have retroactive force

* address comments

* ome more

Author: asurkov

src/main/kotlin/app/hashers/CommitCrawler.kt

@@ -13,6 +13,10 @@ import app.model.Repo
 import app.utils.EmptyRepoException
 import app.utils.FileHelper
 import io.reactivex.Observable
+import java.io.BufferedReader
+import java.io.FileReader
+import java.io.File
+import java.io.InputStreamReader
 import org.apache.commons.codec.digest.DigestUtils
 import org.eclipse.jgit.api.Git
 import org.eclipse.jgit.diff.DiffEntry
@@ -23,6 +27,8 @@ import org.eclipse.jgit.lib.ObjectId
 import org.eclipse.jgit.lib.Repository
 import org.eclipse.jgit.revwalk.RevCommit
 import org.eclipse.jgit.revwalk.RevWalk
+import org.eclipse.jgit.treewalk.filter.PathFilter
+import org.eclipse.jgit.treewalk.TreeWalk
 import org.eclipse.jgit.util.io.DisabledOutputStream
 import java.util.LinkedList
 
@@ -39,6 +45,7 @@ object CommitCrawler {
     private const val LOCAL_HEAD = "HEAD"
     private val REFS = listOf(REMOTE_HEAD, REMOTE_MASTER_BRANCH,
                               LOCAL_MASTER_BRANCH, LOCAL_HEAD)
+    private val CONF_FILE_PATH = ".sourcerer-conf"
     private val MAX_DIFF_SIZE = 600000
 
     fun getDefaultBranchHead(git: Git): ObjectId {
@@ -106,6 +113,18 @@ object CommitCrawler {
         df.setRepository(repo)
         df.isDetectRenames = true
 
+        val confTreeWalk = TreeWalk(repo)
+        confTreeWalk.addTree(head.getTree())
+        confTreeWalk.setFilter(PathFilter.create(CONF_FILE_PATH))
+
+        var ignoredPaths =
+            if (confTreeWalk.next()) {
+                getIgnoredPaths(repo, confTreeWalk.getObjectId(0))
+            }
+            else {
+                listOf()
+            }
+
         var commitCount = 0
         revWalk.markStart(head)
         var commit: RevCommit? = revWalk.next()  // Move the walker to the head.
@@ -167,6 +186,31 @@ object CommitCrawler {
                 }
                 stream != null && !RawText.isBinary(stream)
             }
+            .filter { diff ->
+                val filePath =
+                    if (diff.getNewPath() != DiffEntry.DEV_NULL) {
+                        diff.getNewPath()
+                    } else {
+                        diff.getOldPath()
+                    }
+
+                // Update ignored paths list. The config file has retroactive
+                // force, i.e. if it was added at this commit, then we presume
+                // it is applied to all commits, preceding this commit.
+                if (diff.getOldPath() == CONF_FILE_PATH) {
+                    ignoredPaths =
+                        getIgnoredPaths(repo, diff.getNewId().toObjectId())
+                }
+
+                !ignoredPaths.any { path ->
+                    if (path.endsWith("/")) {
+                        filePath.startsWith(path)
+                    }
+                    else {
+                        path == filePath
+                    }
+                }
+            }
             .map { diff ->
                 JgitDiff(diff, df.toFileHeader(diff).toEditList())
             }
@@ -264,4 +308,39 @@ object CommitCrawler {
             listOf()
         }
     }
+
+    /**
+     * Return a list of paths that should be ignored in commit analysis.
+     */
+    private fun getIgnoredPaths(repo: Repository, objectId: ObjectId?): List<String> {
+        return try {
+            if (objectId == null) {
+                return listOf()
+            }
+
+            val list = mutableListOf<String>()
+            val fileLoader = repo.open(objectId)
+            val reader =
+                BufferedReader(InputStreamReader(fileLoader.openStream()))
+            var collectIgnored = false
+            for (line in reader.lines()) {
+                if (line == "" || line.startsWith("#")) {
+                    continue
+                }
+
+                if (line.startsWith("[")) {
+                    collectIgnored = (line == "[ignore]")
+                    continue
+                }
+
+                if (collectIgnored) {
+                    list.add(line)
+                }
+            }
+            list
+        }
+        catch(e: Exception) {
+            listOf()
+        }
+    }
 }

src/test/kotlin/test/tests/hashers/IgnorePathsTest.kt

@@ -0,0 +1,100 @@
+// Copyright 2018 Sourcerer Inc. All Rights Reserved.
+// Author: Alexander Surkov ([email protected])
+
+package test.tests.hashers
+
+import app.api.MockApi
+import app.extractors.Extractor
+import app.hashers.CommitHasher
+import app.hashers.CommitCrawler
+import app.model.*
+import app.utils.RepoHelper
+import org.eclipse.jgit.api.Git
+import org.jetbrains.spek.api.Spek
+import org.jetbrains.spek.api.dsl.given
+import org.jetbrains.spek.api.dsl.it
+import test.utils.TestRepo
+import java.io.File
+import java.util.stream.StreamSupport.stream
+import kotlin.streams.toList
+import kotlin.test.assertEquals
+
+class IgnorePathsTest : Spek({
+    fun cleanRepos() {
+        Runtime.getRuntime().exec("src/test/delete_repo.sh").waitFor()
+    }
+
+    val userName = "Contributor"
+    val userEmail = "[email protected]"
+
+    // Creation of test repo.
+    cleanRepos()
+
+    given("commits with syntax stats") {
+        val lines = listOf("x = [i**2 for i range(9999)]", "def fn()", "x = 1",
+                "x = map(lambda x: x**2, range(9999))",
+                "x = map(lambda x: x**2, map(lambda x: x**3, range(10))",
+                "x = map(lambda x: x**2, range(10))," +
+                        "map(lambda x: x**3, range(10)))")
+
+        val author = Author(userName, userEmail)
+        val emails = hashSetOf(userEmail)
+
+        val testRepoPath = "../IgnorePaths_t1"
+        val testRepo = TestRepo(testRepoPath)
+
+        val testRehash = "rehash_IgnorePaths_t1"
+        val serverRepo = Repo(rehash = testRehash)
+
+        val mockApi = MockApi(mockRepo = serverRepo)
+        val observable = CommitCrawler.getObservable(testRepo.git, serverRepo)
+
+        it("t1") {
+            testRepo.createFile("test.py", lines)
+            testRepo.commit(message = "commit1", author = author)
+
+            testRepo.createFile("ignore.py", lines)
+            testRepo.commit(message = "commit2", author = author)
+
+            // Add config, ignore.py from previous commit should be
+            // ignored for stats.
+            testRepo.createFile(".sourcerer-conf",
+                                listOf("[ignore]", "ignore.py", "#test.py"))
+            testRepo.commit(message = "commit3", author = author)
+
+            // Uncomment test.py file in config and delete it. The change
+            // should be ignored for statistics.
+            testRepo.deleteLines(".sourcerer-conf", 1, 1)
+            testRepo.insertLines(".sourcerer-conf", 1, listOf("test.py"))
+            testRepo.commit(message = "commit4", author = author)
+
+            testRepo.deleteFile("test.py")
+            testRepo.commit(message = "commit5", author = author)
+
+            val errors = mutableListOf<Throwable>()
+            CommitHasher(serverRepo, mockApi, listOf("rehashes"), emails)
+                .updateFromObservable(observable, { e -> errors.add(e) })
+            if (errors.size > 0) {
+                println(errors[0].message)
+            }
+            assertEquals(0, errors.size)
+
+            val syntaxStats = mockApi.receivedAddedCommits
+                .fold(mutableListOf<CommitStats>()) { allStats, commit ->
+                    allStats.addAll(commit.stats)
+                    allStats
+                }.filter { it.type == Extractor.TYPE_SYNTAX }
+
+            val mapStats = syntaxStats.filter { it.tech == "python>map" }
+            assertEquals(1, mapStats.size)
+            assertEquals(5, mapStats.map { it.numLinesAdded }.sum())
+            assertEquals(0, mapStats.map { it.numLinesDeleted }.sum())
+        }
+
+        afterGroup {
+            testRepo.destroy()
+        }
+    }
+
+    cleanRepos()
+})