feat: borrow heuristics from linguist (#274)

* feat: borrow heuristics from linguist

* address comments

* fix

Author: asurkov

src/main/kotlin/app/extractors/CExtractor.kt

@@ -10,7 +10,6 @@ import app.model.DiffFile
 class CExtractor : ExtractorInterface {
     companion object {
         const val LANGUAGE_NAME = "c"
-        val FILE_EXTS = listOf("c")
         val evaluator by lazy {
             ExtractorInterface.getLibraryClassifier(LANGUAGE_NAME)
         }

src/main/kotlin/app/extractors/CSharpExtractor.kt

@@ -10,7 +10,6 @@ import app.model.DiffFile
 class CSharpExtractor : ExtractorInterface {
     companion object {
         val LANGUAGE_NAME = "csharp"
-        val FILE_EXTS = listOf("cs")
         val LIBRARIES = ExtractorInterface.getLibraries("cs")
         val evaluator by lazy {
             ExtractorInterface.getLibraryClassifier(LANGUAGE_NAME)

src/main/kotlin/app/extractors/CommonExtractor.kt

@@ -6,86 +6,9 @@ package app.extractors
 import app.model.CommitStats
 import app.model.DiffFile
 
-class CommonExtractor : ExtractorInterface {
-    companion object {
-        val FILE_EXTS_MAP = lazy {
-            val reversedMap = mutableMapOf<String, List<String>>()
-            reversedMap["actionscript"] = listOf("as")
-            reversedMap["arduino"] = listOf("ino")
-            reversedMap["assembly"] = listOf("asm", "s", "S")
-            reversedMap["clojure"] = listOf("clj", "cljs", "cljc", "edn")
-            reversedMap["cobol"] = listOf("cbl", "cob", "cpy")
-            reversedMap["coffeescript"] = listOf("coffee", "litcoffee")
-            reversedMap["cuda"] = listOf("cu", "cuh")
-            reversedMap["d"] = listOf("d")
-            reversedMap["dosbatch"] = listOf("bat")
-            reversedMap["emacslisp"] = listOf("el", "elc")
-            reversedMap["erlang"] = listOf("erl", "hrl")
-            reversedMap["elixir"] = listOf("ex", "exs")
-            reversedMap["elm"] = listOf("elm")
-            reversedMap["factor"] = listOf("factor")
-            reversedMap["forth"] = listOf("forth", "4TH")
-            reversedMap["fortran"] = listOf("f", "for", "f90", "f95", "f03",
-                    "f08", "f15")
-            reversedMap["gradle"] = listOf("gradle")
-            reversedMap["groovy"] = listOf("groovy")
-            reversedMap["haskell"] = listOf("hs", "lhs")
-            reversedMap["haxe"] = listOf("hx")
-            reversedMap["html"] = listOf("html", "htm")
-            reversedMap["hy"] = listOf("hy")
-            reversedMap["j"] = listOf("ijs")
-            reversedMap["julia"] = listOf("jl")
-            reversedMap["lisp"] = listOf("lisp", "lsp", "l")
-            reversedMap["lua"] = listOf("lua")
-            reversedMap["makefile"] = listOf("makefile")
-            reversedMap["matlab"] = listOf("m", "mlx")
-            reversedMap["maven"] = listOf("pom")
-            reversedMap["ocaml"] = listOf("ml", "mli")
-            reversedMap["oxygene"] = listOf("oxygene")
-            reversedMap["pascal"] = listOf("pas")
-            reversedMap["perl"] = listOf("pl", "PL")
-            reversedMap["powershell"] = listOf("ps1", "psm1", "psd1")
-            reversedMap["processing"] = listOf("pde")
-            reversedMap["prolog"] = listOf("P")
-            reversedMap["puppet"] = listOf("pp")
-            reversedMap["qml"] = listOf("qml")
-            reversedMap["r"] = listOf("r", "R")
-            reversedMap["rust"] = listOf("rs")
-            reversedMap["sas"] = listOf("sas")
-            reversedMap["scala"] = listOf("scala", "sc")
-            reversedMap["scheme"] = listOf("scm", "ss")
-            reversedMap["shell"] = listOf("sh")
-            reversedMap["smalltalk"] = listOf("st")
-            reversedMap["sql"] = listOf("sql")
-            reversedMap["tcl"] = listOf("tcl")
-            reversedMap["tex"] = listOf("tex")
-            reversedMap["typescript"] = listOf("ts", "tsx")
-            reversedMap["verilog"] = listOf("v")
-            reversedMap["vhdl"] = listOf("vhdl")
-            reversedMap["viml"] = listOf("vim")
-            reversedMap["visualbasic"] = listOf("bas")
-            reversedMap["visualbasicforapps"] = listOf("vba")
-            reversedMap["vue"] = listOf("vue")
-            reversedMap["wolframlanguage"] = listOf("nb","m")
-            reversedMap["xtend"] = listOf("xtend")
-
-            val map = hashMapOf<String, String>()
-            reversedMap.forEach({ lang, exts ->
-                exts.forEach { ext -> map.put(ext, lang)}
-            })
-            map
-        }
-    }
-
+class CommonExtractor(val languageName: String) : ExtractorInterface {
     override fun extract(files: List<DiffFile>): List<CommitStats> {
-        files.mapNotNull { file ->
-            val lang = FILE_EXTS_MAP.value[file.extension]
-            if (lang != null) {
-                file.language = lang
-                file
-            } else null
-        }
-
+        files.map { file -> file.language = languageName }
         return super.extract(files)
     }
 }

src/main/kotlin/app/extractors/CppExtractor.kt

@@ -10,7 +10,6 @@ import app.model.DiffFile
 class CppExtractor : ExtractorInterface {
     companion object {
         val LANGUAGE_NAME = "cpp"
-        val FILE_EXTS = listOf("cc", "cpp", "cxx", "c++")
         val evaluator by lazy {
             ExtractorInterface.getLibraryClassifier(LANGUAGE_NAME)
         }

src/main/kotlin/app/extractors/Extractor.kt

@@ -17,51 +17,20 @@ class Extractor : ExtractorInterface {
         val RESTRICTED_EXTS = listOf(".min.js")
 
         fun getAllExtensions(): HashSet<String> {
-            val set =
-                CommonExtractor.FILE_EXTS_MAP.value.keys +
-                CExtractor.FILE_EXTS +
-                CppExtractor.FILE_EXTS +
-                CSharpExtractor.FILE_EXTS +
-                CssExtractor.FILE_EXTS +
-                FSharpExtractor.FILE_EXTS +
-                GoExtractor.FILE_EXTS +
-                JavaExtractor.FILE_EXTS +
-                JavascriptExtractor.FILE_EXTS +
-                KotlinExtractor.FILE_EXTS +
-                ObjectiveCExtractor.FILE_EXTS +
-                PhpExtractor.FILE_EXTS +
-                PythonExtractor.FILE_EXTS +
-                RubyExtractor.FILE_EXTS +
-                SwiftExtractor.FILE_EXTS
-
-            return set.toHashSet()
-        }
-    }
-
-    fun create(extension: String): ExtractorInterface {
-        return when (extension) {
-            in JavascriptExtractor.FILE_EXTS -> JavascriptExtractor()
-            in JavaExtractor.FILE_EXTS -> JavaExtractor()
-            in PythonExtractor.FILE_EXTS -> PythonExtractor()
-            in RubyExtractor.FILE_EXTS -> RubyExtractor()
-            in PhpExtractor.FILE_EXTS -> PhpExtractor()
-            in CExtractor.FILE_EXTS -> CExtractor()
-            in CppExtractor.FILE_EXTS -> CppExtractor()
-            in CSharpExtractor.FILE_EXTS -> CSharpExtractor()
-            in FSharpExtractor.FILE_EXTS -> FSharpExtractor()
-            in GoExtractor.FILE_EXTS -> GoExtractor()
-            in ObjectiveCExtractor.FILE_EXTS -> ObjectiveCExtractor()
-            in SwiftExtractor.FILE_EXTS -> SwiftExtractor()
-            in KotlinExtractor.FILE_EXTS -> KotlinExtractor()
-            in CssExtractor.FILE_EXTS -> CssExtractor()
-            else -> CommonExtractor()
+            return Heuristics
+                .map { (ext, _) -> ext }
+                .toHashSet()
         }
     }
 
     override fun extract(files: List<DiffFile>): List<CommitStats> {
-        return files.groupBy { file -> file.extension }
-            .filter { (extension, _) -> !RESTRICTED_EXTS.contains(extension) }
-            .map { (extension, files) -> create(extension).extract(files) }
+        return files
+            .filter { file -> !RESTRICTED_EXTS.contains(file.extension) }
+            .mapNotNull { file ->
+                val extractor = Heuristics.get(file.extension)
+                if (extractor != null) extractor(file.new.content)?.extract(listOf(file))
+                else null
+            }
             .fold(mutableListOf()) { accStats, stats ->
                 accStats.addAll(stats)
                 accStats

src/main/kotlin/app/extractors/FSharpExtractor.kt

@@ -10,7 +10,6 @@ import app.model.DiffFile
 class FSharpExtractor : ExtractorInterface {
     companion object {
         val LANGUAGE_NAME = "fsharp"
-        val FILE_EXTS = listOf("fs", "fsx")
         // The behaviour of csharp library classifier is the same as for csharp.
         val LIBRARIES = ExtractorInterface.getLibraries("cs")
         val evaluator by lazy {

src/main/kotlin/app/extractors/GoExtractor.kt

@@ -10,7 +10,6 @@ import app.model.DiffFile
 class GoExtractor : ExtractorInterface {
     companion object {
         val LANGUAGE_NAME = "go"
-        val FILE_EXTS = listOf("go")
         val evaluator by lazy {
             ExtractorInterface.getLibraryClassifier(LANGUAGE_NAME)
         }