feat: kotlin clf (#130) (APP-149) (#148)

* wip: add kotlin extractor

* wip: kotlin clf tests

* wip: fix pr

Author: yaronskaya

src/main/kotlin/app/extractors/Extractor.kt

@@ -28,6 +28,7 @@ class Extractor : ExtractorInterface {
             in GoExtractor.FILE_EXTS -> GoExtractor()
             in ObjectiveCExtractor.FILE_EXTS -> ObjectiveCExtractor()
             in SwiftExtractor.FILE_EXTS -> SwiftExtractor()
+            in KotlinExtractor.FILE_EXTS -> KotlinExtractor()
             else -> CommonExtractor()
         }
     }

src/main/kotlin/app/extractors/JavaExtractor.kt

@@ -79,8 +79,10 @@ class JavaExtractor : ExtractorInterface {
     override fun tokenize(line: String): List<String> {
         val importRegex = Regex("""^(.*import)\s[^\n]*""")
         val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
+        val packageRegex = Regex("""^(.*package)\s[^\n]*""")
         var newLine = importRegex.replace(line, "")
         newLine = commentRegex.replace(newLine, "")
+        newLine = packageRegex.replace(newLine, "")
         return super.tokenize(newLine)
     }
 

src/main/kotlin/app/extractors/KotlinExtractor.kt

@@ -0,0 +1,58 @@
+// Copyright 2017 Sourcerer Inc. All Rights Reserved.
+// Author: Liubov Yaronskaya ([email protected])
+
+package app.extractors
+
+import app.model.CommitStats
+import app.model.DiffFile
+
+class KotlinExtractor : ExtractorInterface {
+    companion object {
+        val LANGUAGE_NAME = "kotlin"
+        val FILE_EXTS = listOf("kt")
+        val LIBRARIES = ExtractorInterface.getLibraries(LANGUAGE_NAME)
+        val evaluator by lazy {
+            ExtractorInterface.getLibraryClassifier(LANGUAGE_NAME)
+        }
+    }
+
+    override fun extract(files: List<DiffFile>): List<CommitStats> {
+        files.map { file -> file.language = LANGUAGE_NAME }
+        return super.extract(files)
+    }
+
+    override fun extractImports(fileContent: List<String>): List<String> {
+        val imports = mutableSetOf<String>()
+
+        val regex = Regex("""import\s+(\w+[.\w+]*)""")
+        fileContent.forEach {
+            val res = regex.find(it)
+            if (res != null) {
+                val importedName = res.groupValues[1]
+                LIBRARIES.forEach { library ->
+                    if (importedName.startsWith(library)) {
+                        imports.add(library)
+                    }
+                }
+            }
+        }
+
+        return imports.toList()
+    }
+
+    override fun tokenize(line: String): List<String> {
+        val importRegex = Regex("""^(.*import)\s[^\n]*""")
+        val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
+        val packageRegex = Regex("""^(.*package)\s[^\n]*""")
+        var newLine = importRegex.replace(line, "")
+        newLine = commentRegex.replace(newLine, "")
+        newLine = packageRegex.replace(newLine, "")
+        return super.tokenize(newLine)
+    }
+
+    override fun getLineLibraries(line: String,
+                                  fileLibraries: List<String>): List<String> {
+
+        return super.getLineLibraries(line, fileLibraries, evaluator, LANGUAGE_NAME)
+    }
+}

src/main/resources/data/libraries/kotlin_libraries.txt

@@ -0,0 +1,56 @@
+com.winterbe.expekt
+com.github.vassilibykov.adventkt
+me.lazmaid.kraph
+com.github.kittinunf.fuel
+imgui
+org.mapdb
+uy.kohesive.kovert
+io.reactivex.rxkotlin
+com.compass.snail
+io.kweb
+com.github.andrewoma.kwery
+com.github.pgutkowski.kgraphql
+io.vertx
+org.spekframework.spek2
+com.oneeyedmen.konsent
+com.github.salomonbrys.kotson
+org.jire.kton
+io.kotlintest
+ktx
+org.kotlinprimavera
+org.kottpd
+com.almasb.fxgl
+org.wasabifx.wasabi
+com.fboldog.ext4klaxon
+net.yested
+ua.com.lavi.komock
+kotlinx.nosql
+spark
+com.github.kittinunf.result
+com.nivabit.kuery
+io.thelandscape.krawler
+io.tekniq
+com.beust.klaxon
+com.github.shyiko.levelkt
+io.javalin
+kategory
+tornadofx
+kotliquery
+khttp
+ovr
+com.hexagonkt
+com.nhaarman.mockito_kotlin
+io.ktor
+com.codepoetics.klenses
+io.polymorphicpanda.kspec
+com.natpryce.hamkrest
+org.http4k
+com.vaadin
+io.mockk
+kara
+org.funktionale
+com.github.fluidsonic.fluid.json
+com.squareup.sqldelight
+org.amshove.kluent
+com.danneu.kog
+org.jetbrains.exposed

src/test/kotlin/test/tests/extractors/ExtractorTest.kt

@@ -104,6 +104,12 @@ class ExtractorTest : Spek({
             assertExtractsLineLibraries("grpc",
                     line, CExtractor())
         }
+
+        it("kotlin extractor extracts the library") {
+            val line = "FuelManager.instance.apply {"
+            assertExtractsLineLibraries("com.github.kittinunf.fuel",
+                    line, KotlinExtractor())
+        }
     }
 
     given("code line doesn't use libraries" ) {
@@ -161,6 +167,11 @@ class ExtractorTest : Spek({
             val line = "int main(int argc, char **argv) {"
             assertExtractsNoLibraries(line, CExtractor())
         }
+
+        it("kotlin extractor returns empty list") {
+            val line = "val password = \"P@\$\\\$vv0|2|)\""
+            assertExtractsNoLibraries(line, KotlinExtractor())
+        }
     }
 
     given("import name.h") {
@@ -179,6 +190,12 @@ class ExtractorTest : Spek({
         }
     }
 
+    given("line contains import") {
+        it("kotlin extractor extracts import") {
+            val line = "import kategory.optics.*"
+            val lib = "kategory"
+            assertExtractsImport(lib, line, KotlinExtractor())
+
     given("import cv2 or cv") {
         it("imports opencv") {
             val lib = "opencv"