feat: add repo facts, add email filtering for all hashers (#40)

* feat: add repo facts, add email filtering for all hashers

* chore(CommitHasher): update filters

* fix: merge

Author: astansler

src/main/kotlin/app/FactCodes.kt

@@ -5,4 +5,7 @@ object FactCodes {
     val COMMITS_DAY_TIME = 2
     val LINE_LONGEVITY = 3
     val LINE_LONGEVITY_REPO = 4
+    val REPO_DATE_START = 5
+    val REPO_DATE_END = 6
+    val REPO_TEAM_SIZE = 7
 }

src/main/kotlin/app/hashers/CodeLongevity.kt

@@ -7,7 +7,6 @@ import app.FactCodes
 import app.Logger
 import app.api.Api
 import app.model.Author
-import app.model.LocalRepo
 import app.model.Repo
 import app.model.Fact
 import app.utils.RepoHelper
@@ -106,9 +105,9 @@ class CodeLine(val repo: Repository,
 /**
  * Used to compute age of code lines in the repo.
  */
-class CodeLongevity(private val localRepo: LocalRepo,
-                    private val serverRepo: Repo,
+class CodeLongevity(private val serverRepo: Repo,
                     private val api: Api,
+                    private val emails: HashSet<String>,
                     git: Git) {
     val repo: Repository = git.repository
     val head: RevCommit =
@@ -121,9 +120,6 @@ class CodeLongevity(private val localRepo: LocalRepo,
     }
 
     fun update() {
-        // TODO(anatoly): Add emails from server or hashAll.
-        val emails = hashSetOf(localRepo.author.email)
-
         val sums: MutableMap<String, Long> = emails.associate { Pair(it, 0L) }
                                                    .toMutableMap()
         val totals: MutableMap<String, Int> = emails.associate { Pair(it, 0) }
@@ -150,7 +146,7 @@ class CodeLongevity(private val localRepo: LocalRepo,
         val stats = mutableListOf<Fact>()
         stats.add(Fact(repo = serverRepo,
                        code = FactCodes.LINE_LONGEVITY_REPO,
-                       value = repoAvg.toDouble()))
+                       value = repoAvg.toString()))
         val repoAvgDays = repoAvg / secondsInDay
         Logger.info("Repo average code line age is $repoAvgDays days, "
               + "lines total: $repoTotal")
@@ -160,13 +156,8 @@ class CodeLongevity(private val localRepo: LocalRepo,
             val avg = if (total > 0) { sums[email]!! / total } else 0
             stats.add(Fact(repo = serverRepo,
                            code = FactCodes.LINE_LONGEVITY,
-                           value = avg.toDouble(),
+                           value = avg.toString(),
                            author = Author(email = email)))
-            if (email == localRepo.author.email) {
-                val avgDays = avg / secondsInDay
-                Logger.info("Your average code line age is $avgDays days, "
-                      + "lines total: $total")
-            }
         }
 
         if (stats.size > 0) {

src/main/kotlin/app/hashers/CommitHasher.kt

@@ -7,18 +7,17 @@ import app.Logger
 import app.api.Api
 import app.extractors.Extractor
 import app.model.Commit
-import app.model.LocalRepo
 import app.model.Repo
 import io.reactivex.Observable
 import java.util.concurrent.TimeUnit
 
 /**
  * CommitHasher hashes repository and uploads stats to server.
  */
-class CommitHasher(private val localRepo: LocalRepo,
-                   private val serverRepo: Repo = Repo(),
+class CommitHasher(private val serverRepo: Repo = Repo(),
                    private val api: Api,
-                   private val rehashes: List<String>) {
+                   private val rehashes: List<String>,
+                   private val emails: HashSet<String>) {
 
     init {
         // Delete locally missing commits from server. If found at least one
@@ -30,36 +29,6 @@ class CommitHasher(private val localRepo: LocalRepo,
         deleteCommitsOnServer(deletedCommits)
     }
 
-    private fun findFirstOverlappingCommitRehash(): String? {
-
-        val serverHistoryRehashes = serverRepo.commits
-                                              .map { commit -> commit.rehash }
-                                              .toHashSet()
-        return rehashes.firstOrNull { rehash ->
-            serverHistoryRehashes.contains(rehash)
-        }
-    }
-
-    private fun postCommitsToServer(commits: List<Commit>) {
-        if (commits.isNotEmpty()) {
-            api.postCommits(commits)
-            Logger.debug("Sent ${commits.size} added commits to server")
-        }
-    }
-
-    private fun deleteCommitsOnServer(commits: List<Commit>) {
-        if (commits.isNotEmpty()) {
-            api.deleteCommits(commits)
-            Logger.debug("Sent ${commits.size} deleted commits to server")
-        }
-    }
-
-    private val emailFilter: (Commit) -> Boolean = {
-        val email = it.author.email
-        localRepo.hashAllContributors || (email == localRepo.author.email ||
-            serverRepo.emails.contains(email))
-    }
-
     // Hash added and missing server commits and send them to server.
     fun updateFromObservable(observable: Observable<Commit>,
                              onError: (Throwable) -> Unit) {
@@ -70,10 +39,10 @@ class CommitHasher(private val localRepo: LocalRepo,
             .takeWhile { new ->  // Hash until last known commit.
                 new.rehash != lastKnownCommit?.rehash
             }
-            .filter { commit ->  // Don't hash known.
-                knownCommits.isEmpty() || !knownCommits.contains(commit)
-            }
-            .filter { commit -> emailFilter(commit) }  // Email filtering.
+            // Don't hash known to server commits.
+            .filter { commit -> !knownCommits.contains(commit) }
+            // Hash only commits made by authors with specified emails.
+            .filter { commit -> emails.contains(commit.author.email) }
             .map { commit ->
                 // Mapping and stats extraction.
                 commit.stats = Extractor().extract(commit.diffs)
@@ -95,4 +64,27 @@ class CommitHasher(private val localRepo: LocalRepo,
                 postCommitsToServer(commitsBundle)  // Send ready commits.
             }, onError)
     }
+
+    private fun findFirstOverlappingCommitRehash(): String? {
+        val serverHistoryRehashes = serverRepo.commits
+                                              .map { commit -> commit.rehash }
+                                              .toHashSet()
+        return rehashes.firstOrNull { rehash ->
+            serverHistoryRehashes.contains(rehash)
+        }
+    }
+
+    private fun postCommitsToServer(commits: List<Commit>) {
+        if (commits.isNotEmpty()) {
+            api.postCommits(commits)
+            Logger.debug("Sent ${commits.size} added commits to server")
+        }
+    }
+
+    private fun deleteCommitsOnServer(commits: List<Commit>) {
+        if (commits.isNotEmpty()) {
+            api.deleteCommits(commits)
+            Logger.debug("Sent ${commits.size} deleted commits to server")
+        }
+    }
 }

src/main/kotlin/app/hashers/FactHasher.kt

@@ -9,7 +9,6 @@ import app.api.Api
 import app.model.Author
 import app.model.Commit
 import app.model.Fact
-import app.model.LocalRepo
 import app.model.Repo
 import io.reactivex.Observable
 import java.time.LocalDateTime
@@ -18,63 +17,92 @@ import java.time.ZoneOffset
 /**
  * CommitHasher hashes repository and uploads stats to server.
  */
-class FactHasher(private val localRepo: LocalRepo,
-                 private val serverRepo: Repo = Repo(),
-                 private val api: Api) {
+class FactHasher(private val serverRepo: Repo = Repo(),
+                 private val api: Api,
+                 private val emails: HashSet<String>) {
+    private val fsDayWeek = hashMapOf<String, Array<Int>>()
+    private val fsDayTime = hashMapOf<String, Array<Int>>()
+    private val fsRepoDateStart = hashMapOf<String, Long>()
+    private val fsRepoDateEnd = hashMapOf<String, Long>()
+    private val fsRepoTeamSize = hashSetOf<String>()
 
-    private fun postFactsToServer(facts: List<Fact>) {
-        if (facts.isNotEmpty()) {
-            api.postFacts(facts)
-            Logger.debug("Sent ${facts.size} facts to server")
+    init {
+        for (author in emails) {
+            fsDayWeek.put(author, Array(7) { 0 })
+            fsDayTime.put(author, Array(24) { 0 })
+            fsRepoDateStart.put(author, -1)
+            fsRepoDateEnd.put(author, -1)
         }
     }
 
     fun updateFromObservable(observable: Observable<Commit>,
                              onError: (Throwable) -> Unit) {
-        val factsDayWeek = hashMapOf<Author, Array<Int>>()
-        val factsDayTime = hashMapOf<Author, Array<Int>>()
-
-        // TODO(anatoly): Filter hashing by email as in CommitHasher.
         observable
+            .filter { commit -> emails.contains(commit.author.email) }
             .subscribe({ commit ->  // OnNext.
                 // Calculate facts.
-                val author = commit.author
-                val factDayWeek = factsDayWeek[author] ?: Array(7) { 0 }
-                val factDayTime = factsDayTime[author] ?: Array(24) { 0 }
+                val email = commit.author.email
                 val timestamp = commit.dateTimestamp
                 val dateTime = LocalDateTime.ofEpochSecond(timestamp, 0,
                     ZoneOffset.ofTotalSeconds(commit.dateTimeZoneOffset * 60))
+
+                // DayWeek.
+                val factDayWeek = fsDayWeek[email] ?: Array(7) { 0 }
                 // The value is numbered from 1 (Monday) to 7 (Sunday).
                 factDayWeek[dateTime.dayOfWeek.value - 1] += 1
+                fsDayWeek[email] = factDayWeek
+
+                // DayTime.
+                val factDayTime = fsDayTime[email] ?: Array(24) { 0 }
                 // Hour from 0 to 23.
                 factDayTime[dateTime.hour] += 1
-                factsDayWeek[author] = factDayWeek
-                factsDayTime[author] = factDayTime
+                fsDayTime[email] = factDayTime
+
+                // RepoDateStart.
+                fsRepoDateStart[email] = timestamp
+
+                // RepoDateEnd.
+                if ((fsRepoDateEnd[email] ?: -1) == -1L) {
+                    fsRepoDateEnd[email] = timestamp
+                }
+
+                // RepoTeamSize.
+                fsRepoTeamSize.add(email)
             }, onError, {  // OnComplete.
                 try {
-                    val facts = mutableListOf<Fact>()
-                    factsDayTime.map { (author, list) ->
-                        list.forEachIndexed { hour, count ->
-                            if (count > 0) {
-                                facts.add(Fact(serverRepo,
-                                    FactCodes.COMMITS_DAY_TIME, hour,
-                                    count.toDouble(), author))
-                            }
-                        }
-                    }
-                    factsDayWeek.map { (author, list) ->
-                        list.forEachIndexed { day, count ->
-                            if (count > 0) {
-                                facts.add(Fact(serverRepo,
-                                    FactCodes.COMMITS_DAY_WEEK, day,
-                                    count.toDouble(), author))
-                            }
-                        }
-                    }
-                    postFactsToServer(facts)
+                    postFactsToServer(createFacts())
                 } catch (e: Throwable) {
                     onError(e)
                 }
             })
     }
+
+    private fun createFacts(): List<Fact> {
+        val fs = mutableListOf<Fact>()
+        emails.forEach { email ->
+            val author = Author(email = email)
+            fsDayTime[email]?.forEachIndexed { hour, count -> if (count > 0) {
+                fs.add(Fact(serverRepo, FactCodes.COMMITS_DAY_TIME, hour,
+                            count.toString(), author))
+            }}
+            fsDayWeek[email]?.forEachIndexed { day, count -> if (count > 0) {
+                fs.add(Fact(serverRepo, FactCodes.COMMITS_DAY_WEEK, day,
+                            count.toString(), author))
+            }}
+            fs.add(Fact(serverRepo, FactCodes.REPO_DATE_START, 0,
+                        fsRepoDateStart[email].toString(), author))
+            fs.add(Fact(serverRepo, FactCodes.REPO_DATE_END, 0,
+                        fsRepoDateEnd[email].toString(), author))
+        }
+        fs.add(Fact(serverRepo, FactCodes.REPO_TEAM_SIZE, 0,
+                    fsRepoTeamSize.size.toString()))
+        return fs
+    }
+
+    private fun postFactsToServer(facts: List<Fact>) {
+        if (facts.isNotEmpty()) {
+            api.postFacts(facts)
+            Logger.debug("Sent ${facts.size} facts to server")
+        }
+    }
 }

src/main/kotlin/app/hashers/RepoHasher.kt

@@ -18,6 +18,7 @@ import org.eclipse.jgit.revwalk.RevWalk
 import java.io.File
 import java.io.IOException
 import java.util.*
+import kotlin.collections.HashSet
 
 class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
                  private  val configurator: Configurator) {
@@ -33,13 +34,15 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
         println("Hashing $localRepo...")
         val git = loadGit(localRepo.path)
         try {
-            val (rehashes, _) = fetchRehashesAndAuthors(git)
+            val (rehashes, emails) = fetchRehashesAndEmails(git)
 
             localRepo.parseGitConfig(git.repository.config)
             if (localRepo.author.email.isBlank()) {
                 throw IllegalStateException("Can't load email from Git config")
             }
 
+            val filteredEmails = filterEmails(emails)
+
             initServerRepo(rehashes.last)
 
             if (!isKnownRepo()) {
@@ -60,16 +63,17 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
             // Hash by all plugins.
             val observable = CommitCrawler.getObservable(git, serverRepo)
                                              .publish()
-            CommitHasher(localRepo, serverRepo, api, rehashes)
+            CommitHasher(serverRepo, api, rehashes, filteredEmails)
                 .updateFromObservable(observable, onError)
-            FactHasher(localRepo, serverRepo, api)
+            FactHasher(serverRepo, api, filteredEmails)
                 .updateFromObservable(observable, onError)
+
             // Start and synchronously wait until all subscribers complete.
             observable.connect()
 
             // TODO(anatoly): CodeLongevity hash from observable.
             try {
-                CodeLongevity(localRepo, serverRepo, api, git).update()
+                CodeLongevity(serverRepo, api, filteredEmails, git).update()
             }
             catch (e: Throwable) {
                 onError(e)
@@ -125,7 +129,7 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
             serverRepo.initialCommitRehash, localRepo)
     }
 
-    private fun fetchRehashesAndAuthors(git: Git):
+    private fun fetchRehashesAndEmails(git: Git):
         Pair<LinkedList<String>, HashSet<String>> {
         val head: RevCommit = RevWalk(git.repository)
             .parseCommit(git.repository.resolve(RepoHelper.MASTER_BRANCH))
@@ -134,17 +138,29 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
         revWalk.markStart(head)
 
         val commitsRehashes = LinkedList<String>()
-        val authors = hashSetOf<String>()
+        val emails = hashSetOf<String>()
 
         var commit: RevCommit? = revWalk.next()
         while (commit != null) {
             commitsRehashes.add(DigestUtils.sha256Hex(commit.name))
-            authors.add(commit.authorIdent.emailAddress)
+            emails.add(commit.authorIdent.emailAddress)
             commit.disposeBody()
             commit = revWalk.next()
         }
         revWalk.dispose()
 
-        return Pair(commitsRehashes, authors)
+        return Pair(commitsRehashes, emails)
+    }
+
+    private fun filterEmails(emails: HashSet<String>): HashSet<String> {
+        if (localRepo.hashAllContributors) {
+            return emails
+        }
+
+        val knownEmails = mutableListOf<String>()
+        knownEmails.add(serverRepo.userEmail)
+        knownEmails.addAll(serverRepo.emails)
+
+        return knownEmails.filter { emails.contains(it) }.toHashSet()
     }
 }