feat: ignore vendor specific files (#298)

* fix: get rid of allowed extensions in commit crawler
* feat: ignore vendor specific files

Author: asurkov

src/main/kotlin/app/extractors/Heuristics.kt

@@ -231,15 +231,6 @@ val XPMRegex = Regex(
  */
 object Heuristics
 {
-    /**
-     * Returns a hash map of all supported file extensions.
-     */
-    fun getAllExtensions(): HashSet<String> {
-        return HeuristicsMap
-            .map { (ext, _) -> ext }
-            .toHashSet()
-    }
-
     /**
      * Returns a list of language commit stats extracted from the given file.
      */

src/main/kotlin/app/hashers/CommitCrawler.kt

@@ -4,6 +4,7 @@
 package app.hashers
 
 import app.Logger
+//import app.hashers.VendorConventions
 import app.model.Author
 import app.model.Commit
 import app.model.DiffContent
@@ -100,8 +101,7 @@ object CommitCrawler {
     fun getJGitObservable(git: Git,
                           totalCommitCount: Int = 0,
                           filteredEmails: HashSet<String>? = null,
-                          tail : RevCommit? = null,
-                          allowedExts: HashSet<String>? = null) :
+                          tail : RevCommit? = null) :
         Observable<JgitPair> = Observable.create { subscriber ->
         val repo: Repository = git.repository
         val revWalk = RevWalk(repo)
@@ -168,9 +168,10 @@ object CommitCrawler {
             }
             .filter { diff ->
                 val path = diff.newPath
-                val ext = FileHelper.getFileExtension(path)
-                if (allowedExts != null && !allowedExts.contains(ext)) {
-                    return@filter false
+                for (cnv in VendorConventions) {
+                    if (cnv.containsMatchIn(path)) {
+                        return@filter false
+                    }
                 }
 
                 val fileId =

src/main/kotlin/app/hashers/RepoHasher.kt

@@ -73,8 +73,7 @@ class RepoHasher(private val api: Api,
                 filteredEmails
             } else null
             val jgitObservable = CommitCrawler.getJGitObservable(git,
-                rehashes.size, crawlerEmails,
-                allowedExts = Heuristics.getAllExtensions()
+                rehashes.size, crawlerEmails
             ).publish()
             val observable = CommitCrawler.getObservable(git,
                 jgitObservable, serverRepo)