feat: add date fun facts (APP-61, APP-49) (#32)

* feat: add day of week and time of day fun facts (APP-49, APP-61)

* wip: add account of time zones, switch to long type for timestamp

* feat: test for date facts, add date to commit in TestRepo

* fix: add FactKey for all key names, fix PR requests

* chore: add year to test

Author: astansler

src/main/kotlin/app/FactKey.kt

@@ -0,0 +1,8 @@
+package app
+
+object FactKey {
+    val COMMITS_DAY_WEEK = "commits-day-week-"
+    val COMMITS_DAY_TIME = "commits-day-time-"
+    val LINE_LONGEVITY = "line-longevity-avg"
+    val LINE_LONGEVITY_REPO = "line-longevity-repo-avg"
+}

src/main/kotlin/app/hashers/CodeLongevity.kt

@@ -3,6 +3,7 @@
 
 package app.hashers
 
+import app.FactKey
 import app.Logger
 import app.api.Api
 import app.config.Configurator
@@ -80,9 +81,6 @@ class CodeLongevity(private val localRepo: LocalRepo,
                     private val serverRepo: Repo,
                     private val api: Api,
                     private val git: Git, tailRev: String = "") {
-    val CODE_LONGEVITY_KEY = "line-longevity-avg"
-    val CODE_LONGEVITY_REPO_KEY = "line-longevity-repo-avg"
-
     val repo: Repository = git.repository
     val head: RevCommit =
         RevWalk(repo).parseCommit(repo.resolve(RepoHelper.MASTER_BRANCH))
@@ -121,7 +119,7 @@ class CodeLongevity(private val localRepo: LocalRepo,
         val repoAvg = if (repoTotal > 0) { repoSum / repoTotal } else 0
         val stats = mutableListOf<Fact>()
         stats.add(Fact(repo = serverRepo,
-                       key = CODE_LONGEVITY_REPO_KEY,
+                       key = FactKey.LINE_LONGEVITY_REPO,
                        value = repoAvg.toDouble()))
         val repoAvgDays = repoAvg / secondsInDay
         Logger.info("Repo average code line age is $repoAvgDays days, "
@@ -131,7 +129,7 @@ class CodeLongevity(private val localRepo: LocalRepo,
             val total = totals[email] ?: 0
             val avg = if (total > 0) { sums[email]!! / total } else 0
             stats.add(Fact(repo = serverRepo,
-                           key = CODE_LONGEVITY_KEY,
+                           key = FactKey.LINE_LONGEVITY,
                            value = avg.toDouble(),
                            author = Author(email = email)))
             if (email == localRepo.author.email) {

src/main/kotlin/app/hashers/CommitHasher.kt

@@ -3,13 +3,16 @@
 
 package app.hashers
 
+import app.FactKey
 import app.Logger
 import app.api.Api
 import app.extractors.Extractor
+import app.model.Author
 import app.model.Commit
 import app.model.DiffContent
 import app.model.DiffFile
 import app.model.DiffRange
+import app.model.Fact
 import app.model.LocalRepo
 import app.model.Repo
 import app.utils.RepoHelper
@@ -26,6 +29,8 @@ import org.eclipse.jgit.lib.ObjectId
 import org.eclipse.jgit.errors.MissingObjectException
 import org.eclipse.jgit.revwalk.RevCommit
 import org.eclipse.jgit.util.io.DisabledOutputStream
+import java.time.LocalDateTime
+import java.time.ZoneOffset
 import java.util.concurrent.TimeUnit
 
 /**
@@ -35,32 +40,52 @@ class CommitHasher(private val localRepo: LocalRepo,
                    private val repo: Repo = Repo(),
                    private val api: Api,
                    private val git: Git) {
+
     private val gitRepo: Repository = git.repository
 
     private fun findFirstOverlappingCommit(): Commit? {
         val serverHistoryCommits = repo.commits.toHashSet()
-        return getObservableCommits()
+        return getCommitsAsObservable()
             .skipWhile { commit -> !serverHistoryCommits.contains(commit) }
             .blockingFirst(null)
     }
 
     private fun hashAndSendCommits() {
         val lastKnownCommit = repo.commits.lastOrNull()
         val knownCommits = repo.commits.toHashSet()
+
+        val factsDayWeek = hashMapOf<Author, Array<Int>>()
+        val factsDayTime = hashMapOf<Author, Array<Int>>()
+
         // Commits are combined in pairs, an empty commit concatenated to
         // calculate the diff of the initial commit.
-        Observable.concat(getObservableCommits(), Observable.just(Commit()))
+        Observable.concat(getCommitsAsObservable()
+            .doOnNext { commit ->
+                Logger.debug("Commit: ${commit.raw?.name ?: ""}: "
+                    + commit.raw?.shortMessage)
+                commit.repo = repo
+
+                // Calculate facts.
+                val author = commit.author
+                val factDayWeek = factsDayWeek[author] ?: Array(7) { 0 }
+                val factDayTime = factsDayTime[author] ?: Array(24) { 0 }
+                val timestamp = commit.dateTimestamp
+                val dateTime = LocalDateTime.ofEpochSecond(timestamp, 0,
+                    ZoneOffset.ofTotalSeconds(commit.dateTimeZoneOffset * 60))
+                // The value is numbered from 1 (Monday) to 7 (Sunday).
+                factDayWeek[dateTime.dayOfWeek.value - 1] += 1
+                // Hour from 0 to 23.
+                factDayTime[dateTime.hour] += 1
+                factsDayWeek[author] = factDayWeek
+                factsDayTime[author] = factDayTime
+            }, Observable.just(Commit()))
             .pairWithNext()  // Pair commits to get diff.
             .takeWhile { (new, _) ->  // Hash until last known commit.
                 new.rehash != lastKnownCommit?.rehash }
             .filter { (new, _) -> knownCommits.isEmpty()  // Don't hash known.
                 || !knownCommits.contains(new) }
             .filter { (new, _) -> emailFilter(new) }  // Email filtering.
             .map { (new, old) ->  // Mapping and stats extraction.
-                Logger.debug("Commit: ${new.raw?.name ?: ""}: "
-                    + new.raw?.shortMessage)
-                new.repo = repo
-
                 val diffFiles = getDiffFiles(new, old)
                 Logger.debug("Diff: ${diffFiles.size} entries")
                 new.stats = Extractor().extract(diffFiles)
@@ -73,18 +98,33 @@ class CommitHasher(private val localRepo: LocalRepo,
                     total + file.getAllAdded().size }
                 new.numLinesDeleted = diffFiles.fold(0) { total, file ->
                     total + file.getAllDeleted().size }
-
                 new
             }
             .observeOn(Schedulers.io())  // Different thread for data sending.
             .buffer(20, TimeUnit.SECONDS)  // Group ready commits by time.
-            .doOnNext { commitsBundle ->  // Send ready commits.
-                postCommitsToServer(commitsBundle) }
-            .blockingSubscribe({
-                // OnNext
-            }, { t ->  // OnError
-                Logger.error("Error while hashing: $t")
-                t.printStackTrace()
+            .blockingSubscribe({ commitsBundle ->  // OnNext.
+                postCommitsToServer(commitsBundle)  // Send ready commits.
+            }, { e ->  // OnError.
+                Logger.error("Error while hashing: $e")
+            }, {  // OnComplete.
+                val facts = mutableListOf<Fact>()
+                factsDayTime.map { (author, list) ->
+                    list.forEachIndexed { hour, count ->
+                        if (count > 0) {
+                            facts.add(Fact(repo, FactKey.COMMITS_DAY_TIME +
+                                hour, count.toDouble(), author))
+                        }
+                    }
+                }
+                factsDayWeek.map { (author, list) ->
+                    list.forEachIndexed { day, count ->
+                        if (count > 0) {
+                            facts.add(Fact(repo, FactKey.COMMITS_DAY_WEEK +
+                                day, count.toDouble(), author))
+                        }
+                    }
+                }
+                postFactsToServer(facts)
             })
     }
 
@@ -142,14 +182,21 @@ class CommitHasher(private val localRepo: LocalRepo,
         }
     }
 
+    private fun postFactsToServer(facts: List<Fact>) {
+        if (facts.isNotEmpty()) {
+            api.postFacts(facts)
+            Logger.debug("Sent ${facts.size} facts to server")
+        }
+    }
+
     private fun deleteCommitsOnServer(commits: List<Commit>) {
         if (commits.isNotEmpty()) {
             api.deleteCommits(commits)
             Logger.debug("Sent ${commits.size} deleted commits to server")
         }
     }
 
-    private fun getObservableCommits(): Observable<Commit> =
+    private fun getCommitsAsObservable(): Observable<Commit> =
         Observable.create { subscriber ->
         try {
             val revWalk = RevWalk(gitRepo)

src/main/kotlin/app/model/Commit.kt

@@ -18,7 +18,8 @@ data class Commit(
         // Tree rehash used for adjustments of stats due to rebase and fraud.
         var treeRehash: String = "",
         var author: Author = Author(),
-        var dateTimestamp: Int = 0,
+        var dateTimestamp: Long = 0,
+        var dateTimeZoneOffset: Int = 0,
         var isQommit: Boolean = false,
         var numLinesAdded: Int = 0,
         var numLinesDeleted: Int = 0,
@@ -33,7 +34,8 @@ data class Commit(
         rehash = DigestUtils.sha256Hex(revCommit.id.name)
         author = Author(revCommit.authorIdent.name,
                         revCommit.authorIdent.emailAddress)
-        dateTimestamp = revCommit.commitTime
+        dateTimestamp = revCommit.authorIdent.getWhen().time / 1000
+        dateTimeZoneOffset = revCommit.authorIdent.timeZoneOffset
         treeRehash = DigestUtils.sha256Hex(revCommit.tree.name)
     }
 

src/main/proto/sourcerer.proto

@@ -24,7 +24,7 @@ message Commit {
     string author_email = 5;
 
     // Timestamp of a commit creation in seconds UTC+0.
-    uint32 date = 6;
+    uint64 date = 6;
 
     // Is quality commit.
     bool is_qommit = 7;

src/test/kotlin/test/tests/hashers/CodeLongevityTest.kt

@@ -12,7 +12,6 @@ import app.model.*
 import test.utils.TestRepo
 
 import kotlin.test.assertEquals
-import kotlin.test.assertNotEquals
 
 import org.eclipse.jgit.revwalk.RevCommit
 
@@ -60,7 +59,7 @@ class CodeLongevityTest : Spek({
         val fileName = "test1.txt"
 
         // t1: initial insertion
-        testRepo.newFile(fileName, listOf("line1", "line2"))
+        testRepo.createFile(fileName, listOf("line1", "line2"))
         val rev1 = testRepo.commit("inital commit")
         val lines1 = CodeLongevity(
             LocalRepo(testRepoPath), Repo(), MockApi(), testRepo.git).compute()
@@ -177,7 +176,7 @@ class CodeLongevityTest : Spek({
           "line17",
           "line18"
         )
-        testRepo.newFile(fileName, fileContent)
+        testRepo.createFile(fileName, fileContent)
         val rev1 = testRepo.commit("inital commit")
         val lines1 = CodeLongevity(
             LocalRepo(testRepoPath), Repo(), MockApi(), testRepo.git).compute()

src/test/kotlin/test/tests/hashers/CommitHasherTest.kt

@@ -4,6 +4,7 @@
 
 package test.tests.hashers
 
+import app.FactKey
 import app.api.MockApi
 import app.hashers.CommitHasher
 import app.model.*
@@ -12,11 +13,14 @@ import org.eclipse.jgit.api.Git
 import org.jetbrains.spek.api.Spek
 import org.jetbrains.spek.api.dsl.given
 import org.jetbrains.spek.api.dsl.it
+import test.utils.TestRepo
 import java.io.File
+import java.util.*
 import java.util.stream.StreamSupport.stream
 import kotlin.streams.toList
 import kotlin.test.assertEquals
 import kotlin.test.assertNotEquals
+import kotlin.test.assertTrue
 
 class CommitHasherTest : Spek({
     val userName = "Contributor"
@@ -56,6 +60,14 @@ class CommitHasherTest : Spek({
         return lastCommit
     }
 
+    fun createDate(year: Int = 2017, month: Int = 1, day: Int = 1,
+                   hour: Int = 0, minute: Int = 0, seconds: Int = 0): Date {
+        val cal = Calendar.getInstance()
+        // Month in calendar is 0-based.
+        cal.set(year, month - 1, day, hour, minute, seconds)
+        return cal.time
+    }
+
     given("repo with initial commit and no history") {
         repo.commits = listOf()
 
@@ -199,5 +211,69 @@ class CommitHasherTest : Spek({
         }
     }
 
+    given("test of commits date facts") {
+        val testRepoPath = "../testrepo1"
+        val testRepo = TestRepo(testRepoPath)
+        val author1 = Author("Test1", "[email protected]")
+        val author2 = Author("Test2", "[email protected]")
+
+        val repo = Repo(rehash = "rehash", commits = listOf())
+        val mockApi = MockApi(mockRepo = repo)
+        val facts = mockApi.receivedFacts
+
+        afterEachTest {
+            facts.clear()
+        }
+
+        it("send two facts") {
+            testRepo.createFile("test1.txt", listOf("line1", "line2"))
+            testRepo.commit(message = "initial commit",
+                            author = author1,
+                            // Sunday.
+                            date = createDate(year = 2017, month = 1, day = 1,
+                                hour = 13, minute = 0, seconds = 0))
+
+            CommitHasher(localRepo, repo, mockApi, testRepo.git).update()
+
+            assertEquals(2, facts.size)
+            assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_TIME + 13,
+                1.0, author1)))
+            assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_WEEK + 6,
+                1.0, author1)))
+        }
+
+        it("send more facts") {
+            testRepo.createFile("test2.txt", listOf("line1", "line2"))
+            testRepo.commit(message = "second commit",
+                            author = author2,
+                            // Monday.
+                            date = createDate(year=2017, month = 1, day = 2,
+                                hour = 18, minute = 0, seconds = 0))
+
+            testRepo.createFile("test3.txt", listOf("line1", "line2"))
+            testRepo.commit(message = "third commit",
+                            author = author1,
+                            // Monday.
+                            date = createDate(month = 1, day = 2,
+                                hour = 13, minute = 0, seconds = 0))
+
+            CommitHasher(localRepo, repo, mockApi, testRepo.git).update()
+
+            assertEquals(5, facts.size)
+            assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_TIME + 18,
+                1.0, author2)))
+            assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_WEEK + 0,
+                1.0, author2)))
+            assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_TIME + 13,
+                2.0, author1)))
+            assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_WEEK + 0,
+                1.0, author1)))
+        }
+
+        afterGroup {
+            testRepo.destroy()
+        }
+    }
+
     Runtime.getRuntime().exec("src/test/delete_repo.sh").waitFor()
 })