feat: add filter of file by extensions (#221)

* feat: add filter of file by extensions

* refactor: move code

* chore: fix indent

Author: anatolystansler

src/main/kotlin/app/Main.kt

@@ -5,6 +5,7 @@ package app
 
 import app.api.ServerApi
 import app.config.FileConfigurator
+import app.extractors.Extractor
 import app.model.LocalRepo
 import app.ui.ConsoleUi
 import app.utils.CommandConfig

src/main/kotlin/app/extractors/Extractor.kt

@@ -15,6 +15,26 @@ class Extractor : ExtractorInterface {
         val TYPE_SYNTAX = 4
         val SEPARATOR = ">"
         val RESTRICTED_EXTS = listOf(".min.js")
+
+        fun getAllExtensions(): HashSet<String> {
+            val set =
+                CommonExtractor.FILE_EXTS_MAP.value.keys +
+                CExtractor.FILE_EXTS +
+                CppExtractor.FILE_EXTS +
+                CSharpExtractor.FILE_EXTS +
+                CssExtractor.FILE_EXTS +
+                GoExtractor.FILE_EXTS +
+                JavaExtractor.FILE_EXTS +
+                JavascriptExtractor.FILE_EXTS +
+                KotlinExtractor.FILE_EXTS +
+                ObjectiveCExtractor.FILE_EXTS +
+                PhpExtractor.FILE_EXTS +
+                PythonExtractor.FILE_EXTS +
+                RubyExtractor.FILE_EXTS +
+                SwiftExtractor.FILE_EXTS
+
+            return set.toHashSet()
+        }
     }
 
     fun create(extension: String): ExtractorInterface {

src/main/kotlin/app/hashers/CommitCrawler.kt

@@ -11,6 +11,7 @@ import app.model.DiffFile
 import app.model.DiffRange
 import app.model.Repo
 import app.utils.EmptyRepoException
+import app.utils.FileHelper
 import io.reactivex.Observable
 import org.apache.commons.codec.digest.DigestUtils
 import org.eclipse.jgit.api.Git
@@ -91,9 +92,9 @@ object CommitCrawler {
     fun getJGitObservable(git: Git,
                           totalCommitCount: Int = 0,
                           filteredEmails: HashSet<String>? = null,
-                          tail : RevCommit? = null) : Observable<JgitPair> =
-        Observable.create { subscriber ->
-
+                          tail : RevCommit? = null,
+                          allowedExts: HashSet<String>? = null) :
+        Observable<JgitPair> = Observable.create { subscriber ->
         val repo: Repository = git.repository
         val revWalk = RevWalk(repo)
         val head: RevCommit =
@@ -146,14 +147,23 @@ object CommitCrawler {
                 diff.changeType != DiffEntry.ChangeType.COPY
             }
             .filter { diff ->
+                val path = diff.newPath
+                val ext = FileHelper.getFileExtension(path)
+                if (allowedExts != null && !allowedExts.contains(ext)) {
+                    return@filter false
+                }
+
                 val fileId =
-                    if (diff.getNewPath() != DiffEntry.DEV_NULL) {
+                    if (path != DiffEntry.DEV_NULL) {
                         diff.getNewId().toObjectId()
                     } else {
                         diff.getOldId().toObjectId()
                     }
-                val stream = try { repo.open(fileId).openStream() }
-                catch (e: Exception) { null }
+                val stream = try {
+                    repo.open(fileId).openStream()
+                } catch (e: Exception) {
+                    null
+                }
                 stream != null && !RawText.isBinary(stream)
             }
             .map { diff ->

src/main/kotlin/app/hashers/RepoHasher.kt

@@ -7,6 +7,7 @@ import app.BuildConfig
 import app.Logger
 import app.api.Api
 import app.config.Configurator
+import app.extractors.Extractor
 import app.model.Author
 import app.model.LocalRepo
 import app.model.ProcessEntry
@@ -69,7 +70,9 @@ class RepoHasher(private val api: Api,
                 filteredEmails
             } else null
             val jgitObservable = CommitCrawler.getJGitObservable(git,
-                rehashes.size, crawlerEmails).publish()
+                rehashes.size, crawlerEmails,
+                allowedExts = Extractor.getAllExtensions()
+            ).publish()
             val observable = CommitCrawler.getObservable(git,
                 jgitObservable, serverRepo)