feat: add sending of meta data for cloud hashing, refactoring (#210)

anatolystansler · web-flow · commit 14c33a8aab5c · 2018-02-19T16:54:40.000+03:00
* feat: add meta model

* chore: fix indent size

* wip: refactor code

* chore: add more codes

* chore: fix todo text

* fix: hidding of commit details with flag
diff --git a/src/main/kotlin/app/Logger.kt b/src/main/kotlin/app/Logger.kt
@@ -150,10 +150,12 @@ object Logger {
     private val commitDetailIndent = generateIndent(10) + "|" +
         generateIndent(8)
     fun printCommitDetail(message: String) {
-        val messageTrim = if (message.length > 59) {
-            message.substring(0, 56).plus("...")
-        } else message
-        println(commitDetailIndent + messageTrim)
+        if (!SILENT) {
+            val messageTrim = if (message.length > 59) {
+                message.substring(0, 56).plus("...")
+            } else message
+            println(commitDetailIndent + messageTrim)
+        }
     }
 
     /**
diff --git a/src/main/kotlin/app/api/Api.kt b/src/main/kotlin/app/api/Api.kt
@@ -14,9 +14,14 @@ import app.model.User
 interface Api {
     companion object {
         val OUT_OF_DATE = 1
+        val STATUS_CLONING = 80
         val PROCESS_STATUS_START = 100
         val PROCESS_STATUS_COMPLETE = 200
         val PROCESS_STATUS_FAIL = 1000
+        val CODE_SUCCESS = 0
+        val PROCESS_ERROR_TOO_MUCH_COMMITS = 4
+        val PROCESS_ERROR_NO_COMMITS = 5
+        val PROCESS_ERROR_PROCESSOR = 6
         val PROCESS_ERROR_EMPTY_REPO = 8
     }
 
diff --git a/src/main/kotlin/app/hashers/RepoHasher.kt b/src/main/kotlin/app/hashers/RepoHasher.kt
@@ -20,38 +20,39 @@ import java.io.File
 import java.io.IOException
 import kotlin.collections.HashSet
 
-class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
-                 private val configurator: Configurator,
-                 private val processEntryId: Int? = null) {
-    var serverRepo: Repo = Repo()
+class RepoHasher(private val api: Api,
+                 private val configurator: Configurator) {
+    fun update(localRepo: LocalRepo) {
+        Logger.debug { "RepoHasher.update call: " + localRepo }
+        val processEntryId = localRepo.processEntryId
 
-    init {
         if (!RepoHelper.isValidRepo(localRepo.path.toPath())) {
+            // TODO(anatoly): Send empty repo.
             throw IllegalArgumentException("Invalid repo $localRepo")
         }
-    }
 
-    fun update() {
-        Logger.info { "Hashing of repo started" }
         val git = loadGit(localRepo.path)
         try {
+            Logger.info { "Hashing of repo started" }
             updateProcess(processEntryId, Api.PROCESS_STATUS_START)
-            val (rehashes, authors) = CommitCrawler.fetchRehashesAndAuthors(git)
 
+            val (rehashes, authors) = CommitCrawler.fetchRehashesAndAuthors(git)
             localRepo.parseGitConfig(git.repository.config)
-            initServerRepo(rehashes.last)
-            Logger.debug { "Local repo path: ${localRepo.path}" }
-            Logger.debug { "Repo remote: ${localRepo.remoteOrigin}" }
-            Logger.debug { "Repo rehash: ${serverRepo.rehash}" }
+            val serverRepo = initServerRepo(localRepo, rehashes.last)
 
             // Get repo setup (commits, emails to hash) from server.
-            postRepoFromServer()
+            postRepoFromServer(serverRepo)
 
             // Send all repo emails for invites.
-            postAuthorsToServer(authors)
+            postAuthorsToServer(authors, serverRepo)
 
+            // Choose emails to filter commits with.
             val emails = authors.map { author -> author.email }.toHashSet()
-            val filteredEmails = filterEmails(emails)
+            val filteredEmails = if (localRepo.hashAllContributors) {
+                emails
+            } else {
+                filterEmails(emails, serverRepo)
+            }
 
             // Common error handling for subscribers.
             // Exceptions can't be thrown out of reactive chain.
@@ -124,7 +125,7 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
         git.close()
     }
 
-    private fun postRepoFromServer() {
+    private fun postRepoFromServer(serverRepo: Repo) {
         val repo = api.postRepo(serverRepo).getOrThrow()
         serverRepo.commits = repo.commits
         Logger.info {
@@ -133,24 +134,26 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api,
         Logger.debug { serverRepo.toString() }
     }
 
-    private fun postAuthorsToServer(authors: HashSet<Author>) {
-        api.postAuthors(authors.map { author ->
-            author.repo = serverRepo
-            author
-        }).onErrorThrow()
+    private fun postAuthorsToServer(authors: HashSet<Author>,
+                                    serverRepo: Repo) {
+        authors.forEach { author -> author.repo = serverRepo }
+        api.postAuthors(authors.toList()).onErrorThrow()
     }
 
-    private fun initServerRepo(initCommitRehash: String) {
-        serverRepo = Repo(initialCommitRehash = initCommitRehash,
-                          rehash = RepoHelper.calculateRepoRehash(
-                              initCommitRehash, localRepo))
+    private fun initServerRepo(localRepo: LocalRepo,
+                               initCommitRehash: String): Repo {
+        val rehash = RepoHelper.calculateRepoRehash(initCommitRehash, localRepo)
+        val repo = Repo(initialCommitRehash = initCommitRehash,
+                        rehash = rehash,
+                        meta = localRepo.meta)
+        Logger.debug { "Local repo path: ${localRepo.path}" }
+        Logger.debug { "Repo remote: ${localRepo.remoteOrigin}" }
+        Logger.debug { "Repo rehash: $rehash" }
+        return repo
     }
 
-    private fun filterEmails(emails: HashSet<String>): HashSet<String> {
-        if (localRepo.hashAllContributors) {
-            return emails
-        }
-
+    private fun filterEmails(emails: HashSet<String>,
+                             serverRepo: Repo): HashSet<String> {
         val knownEmails = hashSetOf<String>()
         knownEmails.addAll(configurator.getUser().emails.map { it.email })
         knownEmails.addAll(serverRepo.emails)
diff --git a/src/main/kotlin/app/model/AuthorGroup.kt b/src/main/kotlin/app/model/AuthorGroup.kt
@@ -22,8 +22,8 @@ data class AuthorGroup(
 
     fun getProto(): Protos.AuthorGroup {
         return Protos.AuthorGroup.newBuilder()
-                .addAllAuthors(authors.map { it -> it.getProto() })
-                .build()
+            .addAllAuthors(authors.map { it -> it.getProto() })
+            .build()
     }
 
     fun serialize(): ByteArray {
diff --git a/src/main/kotlin/app/model/Commit.kt b/src/main/kotlin/app/model/Commit.kt
@@ -60,17 +60,17 @@ data class Commit(
 
     fun getProto(): Protos.Commit {
         return Protos.Commit.newBuilder()
-                .setRehash(rehash)
-                .setRepoRehash(repo.rehash)
-                .setTreeRehash(treeRehash)
-                .setAuthorName(author.name)
-                .setAuthorEmail(author.email)
-                .setDate(dateTimestamp)
-                .setIsQommit(isQommit)
-                .setNumLinesAdded(numLinesAdded)
-                .setNumLinesDeleted(numLinesDeleted)
-                .addAllStats(stats.map { it.getProto() })
-                .build()
+            .setRehash(rehash)
+            .setRepoRehash(repo.rehash)
+            .setTreeRehash(treeRehash)
+            .setAuthorName(author.name)
+            .setAuthorEmail(author.email)
+            .setDate(dateTimestamp)
+            .setIsQommit(isQommit)
+            .setNumLinesAdded(numLinesAdded)
+            .setNumLinesDeleted(numLinesDeleted)
+            .addAllStats(stats.map { it.getProto() })
+            .build()
     }
 
     fun serialize(): ByteArray {
diff --git a/src/main/kotlin/app/model/CommitGroup.kt b/src/main/kotlin/app/model/CommitGroup.kt
@@ -25,8 +25,8 @@ data class CommitGroup(
 
     fun getProto(): Protos.CommitGroup {
         return Protos.CommitGroup.newBuilder()
-                .addAllCommits(commits.map { it -> it.getProto() })
-                .build()
+            .addAllCommits(commits.map { it -> it.getProto() })
+            .build()
     }
 
     fun serialize(): ByteArray {
diff --git a/src/main/kotlin/app/model/CommitStats.kt b/src/main/kotlin/app/model/CommitStats.kt
@@ -31,11 +31,11 @@ data class CommitStats(
 
     fun getProto(): Protos.CommitStats {
         return Protos.CommitStats.newBuilder()
-                .setNumLinesAdded(numLinesAdded)
-                .setNumLinesDeleted(numLinesDeleted)
-                .setType(type)
-                .setTech(tech)
-                .build()
+            .setNumLinesAdded(numLinesAdded)
+            .setNumLinesDeleted(numLinesDeleted)
+            .setType(type)
+            .setTech(tech)
+            .build()
     }
 
     fun serialize(): ByteArray {
diff --git a/src/main/kotlin/app/model/FactGroup.kt b/src/main/kotlin/app/model/FactGroup.kt
@@ -25,8 +25,8 @@ data class FactGroup(
 
     fun getProto(): Protos.FactGroup {
         return Protos.FactGroup.newBuilder()
-                .addAllFacts(stats.map { it -> it.getProto() })
-                .build()
+            .addAllFacts(stats.map { it -> it.getProto() })
+            .build()
     }
 
     fun serialize(): ByteArray {
diff --git a/src/main/kotlin/app/model/LocalRepo.kt b/src/main/kotlin/app/model/LocalRepo.kt
@@ -11,6 +11,8 @@ data class LocalRepo(var path: String = "") {
 
     @JsonIgnore var author: Author = Author()
     @JsonIgnore var remoteOrigin: String = ""
+    @JsonIgnore var meta = RepoMeta()
+    @JsonIgnore var processEntryId: Int? = 0
 
     fun parseGitConfig(config: Config) {
         author = Author(
diff --git a/src/main/kotlin/app/model/Repo.kt b/src/main/kotlin/app/model/Repo.kt
@@ -11,16 +11,19 @@ import java.security.InvalidParameterException
  * Repository.
  */
 data class Repo(
-        // Basic info.
-        var rehash: String = "",
-        var initialCommitRehash: String = "",
+    // Basic info.
+    var rehash: String = "",
+    var initialCommitRehash: String = "",
 
-        // Authors' email filter for hashed commits. If empty list then hash
-        // only commits that created by current user.
-        var emails: List<String> = listOf(),
+    // Authors' email filter for hashed commits. If empty list then hash
+    // only commits that created by current user.
+    var emails: List<String> = listOf(),
 
-        // Raw commits server history. Used to find overlap of commits.
-        var commits: List<Commit> = listOf()
+    // Raw commits server history. Used to find overlap of commits.
+    var commits: List<Commit> = listOf(),
+
+    // Meta info about repo. Is not used for a locally run app.
+    var meta: RepoMeta = RepoMeta()
 ) {
     @Throws(InvalidParameterException::class)
     constructor(proto: Protos.Repo) : this() {
@@ -37,11 +40,12 @@ data class Repo(
 
     fun getProto(): Protos.Repo {
         return Protos.Repo.newBuilder()
-                .setRehash(rehash)
-                .setInitialCommitRehash(rehash)
-                .addAllEmails(emails)
-                .addAllCommits(commits.map { it.getProto() })
-                .build()
+            .setRehash(rehash)
+            .setInitialCommitRehash(rehash)
+            .addAllEmails(emails)
+            .addAllCommits(commits.map { it.getProto() })
+            .setMeta(meta.getProto())
+            .build()
     }
 
     fun serialize(): ByteArray {
diff --git a/src/main/kotlin/app/model/RepoMeta.kt b/src/main/kotlin/app/model/RepoMeta.kt
@@ -0,0 +1,44 @@
+package app.model
+
+import app.Protos
+import com.google.protobuf.InvalidProtocolBufferException
+import java.security.InvalidParameterException
+
+/**
+ * Meta info about repo. Is not used for a locally run app.
+ */
+data class RepoMeta(
+    var ghId: String = "",
+    var name: String = "",
+    var ownerName: String = "",
+    var htmlUrl: String = "",
+    var cloneUrl: String = ""
+) {
+    @Throws(InvalidParameterException::class)
+    constructor(proto: Protos.RepoMeta) : this() {
+        ghId = proto.ghId
+        name = proto.name
+        ownerName = proto.ownerName
+        htmlUrl = proto.htmlUrl
+        cloneUrl = proto.cloneUrl
+    }
+
+    @Throws(InvalidProtocolBufferException::class)
+    constructor(bytes: ByteArray) : this(Protos.RepoMeta.parseFrom(bytes))
+
+    constructor(serialized: String) : this(serialized.toByteArray())
+
+    fun getProto(): Protos.RepoMeta {
+        return Protos.RepoMeta.newBuilder()
+            .setGhId(ghId)
+            .setName(name)
+            .setOwnerName(ownerName)
+            .setHtmlUrl(htmlUrl)
+            .setCloneUrl(cloneUrl)
+            .build()
+    }
+
+    fun serialize(): ByteArray {
+        return getProto().toByteArray()
+    }
+}
diff --git a/src/main/kotlin/app/ui/UpdateRepoState.kt b/src/main/kotlin/app/ui/UpdateRepoState.kt
@@ -6,9 +6,9 @@ package app.ui
 import app.BuildConfig
 import app.hashers.RepoHasher
 import app.Logger
-import app.Protos
 import app.api.Api
 import app.config.Configurator
+import app.model.LocalRepo
 import app.model.ProcessEntry
 import app.utils.HashingException
 import java.util.*
@@ -24,16 +24,15 @@ class UpdateRepoState constructor(private val context: Context,
     override fun doAction() {
         Logger.info { "Hashing started" }
 
+        // TODO(anatoly): Move code to RepoHasher.
         val localRepos = configurator.getLocalRepos()
-        val process = api.postProcessCreate(requestNumEntries = localRepos.size)
-                         .getOrThrow()
-        val processEntryIds = process.entries.map { entry -> entry.id }
-        val heartbeatTimer = runHeartbeatTimer(processEntryIds)
-        for ((index, repo) in localRepos.withIndex()) {
+        assignProcess(localRepos)
+
+        val heartbeatTimer = runHeartbeatTimer(localRepos)
+        for (repo in localRepos) {
             try {
                 Logger.print("Hashing $repo repository...", indentLine = true)
-                val processEntryId = processEntryIds.elementAtOrNull(index)
-                RepoHasher(repo, api, configurator, processEntryId).update()
+                RepoHasher(api, configurator).update(repo)
                 Logger.print("Hashing $repo completed.")
             } catch (e: HashingException) {
                 e.errors.forEach { error ->
@@ -56,8 +55,19 @@ class UpdateRepoState constructor(private val context: Context,
         context.changeState(CloseState())
     }
 
-    private fun runHeartbeatTimer(processEntryIds: List<Int>): Timer {
-        val entries = processEntryIds.map { id -> ProcessEntry(id = id) }
+    // Notify server about processing start and get processing ids.
+    private fun assignProcess(localRepos: List<LocalRepo>) {
+        val process = api.postProcessCreate(requestNumEntries = localRepos.size)
+            .getOrThrow()
+        if (process.entries.isEmpty()) return
+        process.entries.subList(0, localRepos.size).forEachIndexed { index, e ->
+            localRepos[index].processEntryId = e.id
+        }
+    }
+
+    private fun runHeartbeatTimer(localRepos: List<LocalRepo>): Timer {
+        val entries = localRepos.filter { it.processEntryId != null}
+                                .map { ProcessEntry(id = it.processEntryId!!) }
         return fixedRateTimer(
             name = "heartbeat",
             daemon = true,
diff --git a/src/main/kotlin/app/utils/RepoHelper.kt b/src/main/kotlin/app/utils/RepoHelper.kt
@@ -32,7 +32,7 @@ object RepoHelper {
             repository = git.repository
             commitId = CommitCrawler.getDefaultBranchHead(git)
         } catch (e: Exception) {
-           Logger.error(e, "Cannot access repository at specified path")
+            Logger.error(e, "Cannot access repository at specified path")
             return false
         } finally {
             repository?.close()
diff --git a/src/main/proto/sourcerer.proto b/src/main/proto/sourcerer.proto

Original file line number	Diff line number	Diff line change
`@@ -22,8 +22,8 @@ data class AuthorGroup(`
`22`	`22`
`23`	`23`	`fun getProto(): Protos.AuthorGroup {`
`24`	`24`	`return Protos.AuthorGroup.newBuilder()`
`25`		`- .addAllAuthors(authors.map { it -> it.getProto() })`
`26`		`- .build()`
	`25`	`+ .addAllAuthors(authors.map { it -> it.getProto() })`
	`26`	`+ .build()`
`27`	`27`	`}`
`28`	`28`
`29`	`29`	`fun serialize(): ByteArray {`
Original file line number	Diff line number	Diff line change
`@@ -25,8 +25,8 @@ data class CommitGroup(`
`25`	`25`
`26`	`26`	`fun getProto(): Protos.CommitGroup {`
`27`	`27`	`return Protos.CommitGroup.newBuilder()`
`28`		`- .addAllCommits(commits.map { it -> it.getProto() })`
`29`		`- .build()`
	`28`	`+ .addAllCommits(commits.map { it -> it.getProto() })`
	`29`	`+ .build()`
`30`	`30`	`}`
`31`	`31`
`32`	`32`	`fun serialize(): ByteArray {`
Original file line number	Diff line number	Diff line change
`@@ -25,8 +25,8 @@ data class FactGroup(`
`25`	`25`
`26`	`26`	`fun getProto(): Protos.FactGroup {`
`27`	`27`	`return Protos.FactGroup.newBuilder()`
`28`		`- .addAllFacts(stats.map { it -> it.getProto() })`
`29`		`- .build()`
	`28`	`+ .addAllFacts(stats.map { it -> it.getProto() })`
	`29`	`+ .build()`
`30`	`30`	`}`
`31`	`31`
`32`	`32`	`fun serialize(): ByteArray {`