release: 0.2.1

Author: anatolystansler

README.md

@@ -3,7 +3,7 @@
   <a href="https://sourcerer.io"><img src=https://user-images.githubusercontent.com/20287615/34189346-d426d4c2-e4ef-11e7-9da4-cc76a1ed111d.png alt="Sourcerer Logo", width=50></a>
   &nbsp; sourcerer.io
 </h1>
-  An intelligent profile for software engineers. Start with your public GitHub at <a href="https://sourcerer.io/start">https://sourcerer.io/start</a>.
+  A visual profile for software engineers. Start with your public GitHub at <a href="https://sourcerer.io/start">https://sourcerer.io/start</a>. Screenshot below.
 <br>
 
 What is it?

build.gradle

@@ -37,7 +37,7 @@ buildConfig {
 
     // App version.
     buildConfigField 'int', 'VERSION_CODE', '6'
-    buildConfigField 'String', 'VERSION', '0.2.0'
+    buildConfigField 'String', 'VERSION', '0.2.1'
 
     // Logging.
     buildConfigField 'String', 'ENV', project.hasProperty('env') ? env : 'production'
@@ -55,6 +55,11 @@ buildConfig {
     // Models storage path.
     buildConfigField 'String', 'LIBRARY_MODELS_URL', 'https://storage.googleapis.com/sourcerer-app/library-models/v1/'
 
+    // Hashing.
+    buildConfigField 'boolean', 'COMMIT_HASHER_ENABLED', project.hasProperty('commit-hasher-enabled') ? project.property('commit-hasher-enabled').toString() : 'true'
+    buildConfigField 'boolean', 'FACT_HASHER_ENABLED', project.hasProperty('fact-hasher-enabled') ? project.property('fact-hasher-enabled').toString() : 'true'
+    buildConfigField 'boolean', 'LONGEVITY_ENABLED', project.hasProperty('longevity-enabled') ? project.property('longevity-enabled').toString() : 'false'
+
     buildConfig
 }
 

src/main/kotlin/app/extractors/CommonExtractor.kt

@@ -40,7 +40,7 @@ class CommonExtractor : ExtractorInterface {
             reversedMap.put("perl", listOf("pl", "PL"))
             reversedMap.put("powershell", listOf("ps1", "psm1", "psd1"))
             reversedMap.put("processing", listOf("pde"))
-            reversedMap.put("prolog", listOf("pl", "pro", "P"))
+            reversedMap.put("prolog", listOf("pl", "P"))
             reversedMap.put("puppet", listOf("pp"))
             reversedMap.put("r", listOf("r", "R"))
             reversedMap.put("rust", listOf("rs"))

src/main/kotlin/app/hashers/CodeLongevity.kt

@@ -456,22 +456,6 @@ class CodeLongevity(
                 val newId = diff.getNewId().toObjectId()
                 Logger.trace { "old: '$oldPath', new: '$newPath'" }
 
-                // Skip binary files.
-                val fileId = if (newPath != DiffEntry.DEV_NULL) newId else oldId
-                try {
-                    if (RawText.isBinary(repo.open(fileId).openStream())) {
-                        continue
-                    }
-                } catch (e: Exception) {
-                    continue
-                    //TODO(anatoly): better exception handling.
-                }
-
-                // TODO(alex): does it happen in the wilds?
-                if (diff.changeType == DiffEntry.ChangeType.COPY) {
-                    continue
-                }
-
                 // File was deleted, initialize the line array in the files map.
                 if (diff.changeType == DiffEntry.ChangeType.DELETE) {
                     val fileLoader = repo.open(oldId)

src/main/kotlin/app/hashers/CommitCrawler.kt

@@ -57,7 +57,7 @@ object CommitCrawler {
     }
 
     fun fetchRehashesAndAuthors(git: Git):
-        Pair<LinkedList<String>, HashSet<Author>> {
+        Triple<LinkedList<String>, HashSet<Author>, HashMap<String, Int>> {
         val head: RevCommit = RevWalk(git.repository)
             .parseCommit(getDefaultBranchHead(git))
 
@@ -67,6 +67,7 @@ object CommitCrawler {
         val commitsRehashes = LinkedList<String>()
         val emails = hashSetOf<String>()
         val names = hashMapOf<String, String>()
+        val commitsCount = hashMapOf<String, Int>()
 
         var commit: RevCommit? = revWalk.next()
         while (commit != null) {
@@ -81,6 +82,7 @@ object CommitCrawler {
                     names[email] = name
                 }
             }
+            commitsCount[email] = commitsCount.getOrDefault(email, 0) + 1
 
             commit.disposeBody()
             commit = revWalk.next()
@@ -90,11 +92,12 @@ object CommitCrawler {
         val authors = emails.map { email -> Author(names[email]!!, email) }
             .toHashSet()
 
-        return Pair(commitsRehashes, authors)
+        return Triple(commitsRehashes, authors, commitsCount)
     }
 
     fun getJGitObservable(git: Git,
                           totalCommitCount: Int = 0,
+                          filteredEmails: HashSet<String>? = null,
                           tail : RevCommit? = null) : Observable<JgitPair> =
         Observable.create { subscriber ->
 
@@ -138,8 +141,29 @@ object CommitCrawler {
             } else 0.0
             Logger.printCommit(commit.shortMessage, commit.name, perc)
 
+            val email = commit.authorIdent.emailAddress
+            if (filteredEmails != null && !filteredEmails.contains(email)) {
+                commit = parentCommit
+                continue
+            }
+
             val diffEntries = df.scan(parentCommit, commit)
-            val diffEdits = diffEntries.map { diff ->
+            val diffEdits = diffEntries
+            .filter { diff ->
+                diff.changeType != DiffEntry.ChangeType.COPY
+            }
+            .filter { diff ->
+                val fileId =
+                    if (diff.getNewPath() != DiffEntry.DEV_NULL) {
+                        diff.getNewId().toObjectId()
+                    } else {
+                        diff.getOldId().toObjectId()
+                    }
+                val stream = try { repo.open(fileId).openStream() }
+                catch (e: Exception) { null }
+                stream != null && !RawText.isBinary(stream)
+            }
+            .map { diff ->
                 JgitDiff(diff, df.toFileHeader(diff).toEditList())
             }
             subscriber.onNext(JgitPair(commit, diffEdits))
@@ -180,18 +204,6 @@ object CommitCrawler {
     private fun getDiffFiles(jgitRepo: Repository,
                              jgitDiffs: List<JgitDiff>) : List<DiffFile> {
         return jgitDiffs
-            // Skip binary files.
-            .filter { (diff, _) ->
-                val fileId =
-                    if (diff.getNewPath() != DiffEntry.DEV_NULL) {
-                        diff.getNewId().toObjectId()
-                    } else {
-                        diff.getOldId().toObjectId()
-                    }
-                val stream = try { jgitRepo.open(fileId).openStream() }
-                catch (e: Exception) { null }
-                stream != null && !RawText.isBinary(stream)
-            }
             .map { (diff, edits) ->
                 // TODO(anatoly): Can produce exception for large object.
                 // Investigate for size.

src/main/kotlin/app/hashers/RepoHasher.kt

@@ -3,6 +3,7 @@
 
 package app.hashers
 
+import app.BuildConfig
 import app.Logger
 import app.api.Api
 import app.config.Configurator
@@ -56,18 +57,29 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
                 Logger.error(e, "Hashing error")
             }
 
+            // Only code longevity needs to calculate each commit, if it's
+            // disabled then read only author's emails.
+            val crawlerEmails = if (!BuildConfig.LONGEVITY_ENABLED) {
+                filteredEmails
+            } else null
+            val jgitObservable = CommitCrawler.getJGitObservable(git,
+                rehashes.size, crawlerEmails).publish()
+            val observable = CommitCrawler.getObservable(git,
+                jgitObservable, serverRepo)
+
             // Hash by all plugins.
-            val jgitObservable =
-                CommitCrawler.getJGitObservable(git, rehashes.size).publish()
-            val observable =
-                CommitCrawler.getObservable(git, jgitObservable, serverRepo)
-
-            CommitHasher(serverRepo, api, rehashes, filteredEmails)
-                .updateFromObservable(observable, onError)
-            FactHasher(serverRepo, api, rehashes, filteredEmails)
-                .updateFromObservable(observable, onError)
-            CodeLongevity(serverRepo, filteredEmails, git)
-                .updateFromObservable(jgitObservable, onError, api)
+            if (BuildConfig.COMMIT_HASHER_ENABLED) {
+                CommitHasher(serverRepo, api, rehashes, filteredEmails)
+                    .updateFromObservable(observable, onError)
+            }
+            if (BuildConfig.FACT_HASHER_ENABLED) {
+                FactHasher(serverRepo, api, rehashes, filteredEmails)
+                    .updateFromObservable(observable, onError)
+            }
+            if (BuildConfig.LONGEVITY_ENABLED) {
+                CodeLongevity(serverRepo, filteredEmails, git)
+                    .updateFromObservable(jgitObservable, onError, api)
+            }
 
             // Start and synchronously wait until all subscribers complete.
             Logger.print("Stats computation. May take a while...")

src/test/kotlin/test/tests/hashers/ColleaguesTest.kt

@@ -32,7 +32,7 @@ import kotlin.test.assertNotNull
  */
 class ColleaguesTest : Spek({
     given("'colleagues #1'") {
-        val testRepoPath = "../CodeLongevity_colleagues1"
+        val testRepoPath = "../colleagues1"
         val testRepo = TestRepo(testRepoPath)
         val testRehash = "rehash_colleagues1"
         val fileName = "test1.txt"
@@ -88,9 +88,9 @@ class ColleaguesTest : Spek({
     }
 
     given("'colleagues stats'") {
-        val testRepoPath = "../CodeLongevity_colleagues_stats"
+        val testRepoPath = "../colleagues_stats"
         val testRepo = TestRepo(testRepoPath)
-        val testRehash = "rehash_lngstats2"
+        val testRehash = "rehash_colleagues_stats"
         val fileName = "test1.txt"
         val author1 = Author(testRepo.userName, testRepo.userEmail)
         val author2 = Author("Vasya Pupkin", "[email protected]")