Merge pull request #45 from aspyker/issues_labels

Issues labels - PR for issue #43 #43

Author: aspyker

.gitignore

@@ -5,4 +5,7 @@ build/
 *.iws
 *.ipr
 .idea/
+*/out
 osstracker-scrapernetflixapp
+cassandra_data
+es_data

osstracker-scraper/build.gradle

@@ -4,13 +4,14 @@ sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
 dependencies {
-    compile 'org.scala-lang:scala-library:2.11.7'
+    compile 'org.scala-lang:scala-library:2.12.4'
     compile 'org.slf4j:slf4j-api:1.7.13'
     compile 'org.slf4j:slf4j-log4j12:1.7.13'
     compile 'org.kohsuke:github-api:1.92'
-    compile 'com.typesafe.play:play-json_2.11:2.4.5'
+    compile 'com.typesafe.play:play-json_2.12:2.6.8'
     compile 'joda-time:joda-time:2.9.1'
-    compile 'org.rogach:scallop_2.11:0.9.5'
+    compile 'org.rogach:scallop_2.12:3.1.1'
     compile 'com.datastax.cassandra:cassandra-driver-core:3.0.0'
     compile 'org.apache.httpcomponents:httpclient:4.5.1'
+    testCompile 'org.scalatest:scalatest_2.12:3.0.4'
 }

osstracker-scraper/src/main/scala/com/netflix/oss/tools/osstrackerscraper/Conf.scala

@@ -19,6 +19,7 @@ import org.rogach.scallop.ScallopConf
 
 class Conf(args: Seq[String]) extends ScallopConf(args) {
   val action = opt[String](required = true)
+  verify()
 }
 
 object Conf {

osstracker-scraper/src/main/scala/com/netflix/oss/tools/osstrackerscraper/GithubAccess.scala

@@ -20,17 +20,31 @@ import java.util.{Date, Properties}
 
 import com.netflix.oss.tools.osstrackerscraper.OssLifecycle.OssLifecycle
 import org.kohsuke.github._
-import org.slf4j.LoggerFactory
-import play.api.libs.json.{Json, JsObject}
+import org.slf4j.{Logger, LoggerFactory}
+import play.api.libs.json.{JsObject, Json}
+
 import scala.collection.JavaConversions._
 
 case class CommitInfo(numCommits: Int, daysSinceLastCommit: Int, contributorLogins: List[String]) {}
-case class IssuesInfo(val closedIssuesSize: Int, val openIssuesSize: Int, val avgIssues: Int) {}
+case class IssuesInfo(
+   val closedCount: Int,
+   val openCount: Int,
+   val avgDayToClose: Int,
+   val openCountWithNoLabels: Int,
+   val openCountWithLabelBug: Int,
+   val openCountWithLabelDuplicate: Int,
+   val openCountWithLabelEnhancement: Int,
+   val openCountWithLabelHelpWanted: Int,
+   val openCountWithLabelInvalid: Int,
+   val openCountWithLabelQuestion: Int,
+   val openCountWithLabelWontfix: Int,
+   val openCountTrulyOpen: Int
+) {}
 case class PRsInfo(val closedPRsSize: Int, val avgPRs: Int) {}
 
-class GithubAccess(val asOfYYYYMMDD: String, val asOfISO: String) {
+class GithubAccess(val asOfYYYYMMDD: String, val asOfISO: String, val connectToGithub: Boolean) {
   val logger = LoggerFactory.getLogger(getClass)
-  val github: GitHub = GitHub.connect()
+  val github: Option[GitHub] = if (connectToGithub) Some(GitHub.connect()) else None
 
   def getOSSMetaDataOSSLifecycle(repo: GHRepository): OssLifecycle = {
     try {
@@ -61,7 +75,7 @@ class GithubAccess(val asOfYYYYMMDD: String, val asOfISO: String) {
 
     val (commitInfo: CommitInfo, issuesInfo: IssuesInfo, prsInfo: PRsInfo) = if (neverPushed) {
       logger.warn("repo has never been pushed, so providing fake zero counts for issues and pull requests")
-      (CommitInfo(0, 0, List[String]()), IssuesInfo(0, 0, 0), PRsInfo(0, 0))
+      (CommitInfo(0, 0, List[String]()), IssuesInfo(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), PRsInfo(0, 0))
     } else {
       val commitInfo = getCommitInfo(repo)
       val issuesInfo = getIssuesStats(repo)
@@ -79,9 +93,19 @@ class GithubAccess(val asOfYYYYMMDD: String, val asOfISO: String) {
       "stars" -> repo.getWatchers(),
       "numContributors" -> commitInfo.contributorLogins.size,
       "issues" -> Json.obj(
-        "openCount" -> issuesInfo.openIssuesSize,
-        "closedCount" -> issuesInfo.closedIssuesSize,
-        "avgTimeToCloseInDays" -> issuesInfo.avgIssues
+        "openCount" -> issuesInfo.openCount,
+        "openCountOnlyIssues" -> issuesInfo.openCountTrulyOpen,
+        "closedCount" -> issuesInfo.closedCount,
+        "avgTimeToCloseInDays" -> issuesInfo.avgDayToClose,
+        "openCountByStandardTags" -> Json.obj(
+          "bug" -> issuesInfo.openCountWithLabelBug,
+          "helpWanted" -> issuesInfo.openCountWithLabelHelpWanted,
+          "question" -> issuesInfo.openCountWithLabelQuestion,
+          "duplicate" -> issuesInfo.openCountWithLabelDuplicate,
+          "enhancement" -> issuesInfo.openCountWithLabelEnhancement,
+          "invalid" -> issuesInfo.openCountWithLabelInvalid,
+          "wontfix" -> issuesInfo.openCountWithLabelWontfix
+        ),
       ),
       "pullRequests" -> Json.obj(
         "openCount" -> openPullRequests.size(),
@@ -129,22 +153,139 @@ class GithubAccess(val asOfYYYYMMDD: String, val asOfISO: String) {
     PRsInfo(closedPRs.size, avgPRs)
   }
 
-  def getIssuesStats(repo: GHRepository) : IssuesInfo = {
-    val closedIssues = repo.getIssues(GHIssueState.CLOSED).filter(_.getPullRequest == null)
-    val openIssues = repo.getIssues(GHIssueState.OPEN).filter(_.getPullRequest == null)
+  def getIssuesStats(repo: GHRepository): IssuesInfo = {
+    val closedIssues = repo.getIssues(GHIssueState.CLOSED).filter(_.getPullRequest == null).toArray
+    val openIssues = repo.getIssues(GHIssueState.OPEN).filter(_.getPullRequest == null).toArray
+    getIssuesStats(closedIssues, openIssues)
+  }
+
+  def getIssuesStats(closedIssues: Array[GHIssue], openIssues: Array[GHIssue]): IssuesInfo = {
+    val (openCountNoLabels, openCountWithLabelBug, openCountWithLabelDuplicate,
+      openCountWithLabelEnhancement, openCountWithLabelHelpWanted,
+      openCountWithLabelInvalid, openCountWithLabelQuestion, openCountWithLabelWontfix,
+      openCountTrulyOpen) = getIssuesLabelStats(openIssues)
+
+
     val timeToCloseIssue = closedIssues.map(issue => {
       val opened = issue.getCreatedAt()
       val closed = issue.getClosedAt()
       val difference = daysBetween(opened, closed)
       difference
     })
     val sumIssues = timeToCloseIssue.sum
-    val avgIssues = timeToCloseIssue.size match {
+    val avgDaysToCloseIssues = timeToCloseIssue.size match {
       case 0 => 0
       case _ => sumIssues / timeToCloseIssue.size
     }
-    logger.debug(s"avg days to close ${closedIssues.size()} issues = ${avgIssues} days")
-    IssuesInfo(closedIssues.size, openIssues.size, avgIssues)
+    logger.debug(s"avg days to close ${closedIssues.length} issues = ${avgDaysToCloseIssues} days")
+
+    IssuesInfo(closedIssues.size, openIssues.size, avgDaysToCloseIssues, openCountNoLabels, openCountWithLabelBug,
+      openCountWithLabelDuplicate, openCountWithLabelEnhancement,
+      openCountWithLabelHelpWanted, openCountWithLabelInvalid, openCountWithLabelQuestion, openCountWithLabelWontfix,
+      openCountTrulyOpen)
+  }
+
+  def getIssuesLabelStats(openIssues: Array[GHIssue]): (Int, Int, Int, Int, Int, Int, Int, Int, Int) = {
+    val openCountNoLabels = openIssues.count(issue => issue.getLabels.size() == 0)
+    // standard labels that count
+    val openCountWithLabelBug = countLabelForIssues(openIssues, "bug")
+    val openCountWithLabelHelpWanted = countLabelForIssues(openIssues, "help wanted")
+    val openCountWithLabelQuestion = countLabelForIssues(openIssues, "question")
+    // standard labels that dont' count
+    val openCountWithLabelDuplicate = countLabelForIssues(openIssues, "duplicate")
+    val openCountWithLabelEnhancement = countLabelForIssues(openIssues, "enhancement")
+    val openCountWithLabelInvalid = countLabelForIssues(openIssues, "invalid")
+    val openCountWithLabelWontfix = countLabelForIssues(openIssues, "wontfix")
+    val openCountTrulyOpen = countLabelsForTrueIssues(openIssues)
+    (
+      openCountNoLabels, openCountWithLabelBug, openCountWithLabelDuplicate,
+      openCountWithLabelEnhancement, openCountWithLabelHelpWanted,
+      openCountWithLabelInvalid, openCountWithLabelQuestion, openCountWithLabelWontfix,
+      openCountTrulyOpen)
+  }
+
+  def countLabelsForTrueIssues(issues: Array[GHIssue]): Int = {
+    // note that some issues will have bug and enhancement, we need to honor the worst case label (bug)
+    // note that some issues will have bug and invalid, we don't want to double count
+    // so, if no label, count it
+    // for single labels
+    //    if (bug || help wanted || question) count it
+    //    if (duplicate || enhancement || invalid || wont fix) don't count it
+    // for multiple labels
+    //    if (bug || help wanted || question) count it
+    //    if no standard github labels count it
+    val count: Int = issues.count(issue => {
+      val labels = issue.getLabels.toList
+
+      val shouldCount = if (labels.size == 0) {
+        true // no labels so counts
+      } else {
+        if (hasBugOrQuestionLabel(labels)) {
+          true // has bug or question, so counts
+        }
+        else if (hasInvalidOrWontFix(labels)) {
+          false // has invalid or wontfix, so doesn't count
+        }
+        else {
+          val duplicate = hasLabelOfName(labels, "duplicate")
+          val enhancement = hasLabelOfName(labels, "enhancement")
+          val helpwanted = hasLabelOfName(labels, "helpwanted")
+          // by this point bug and question and invalid and wontfix = false
+          val computed = (duplicate, enhancement, helpwanted) match {
+            case (false, false, false) => true // no labels except custom labels
+            case (false, false, true) => true // help wanted and [custom labels]
+            case (false, true, false) => false // enhancement and [custom labels]
+            case (false, true, true) => false // enhancement and helpwanted and [custom labels]
+            case (true, false, false) => true // duplicate and [custom labels]
+            case (true, false, true) => true // duplicate and helpwanted and [custom labels]
+            case (true, true, false) => false // duplicate and enhancement and [custom labels]
+            case (true, true, true) => false // duplicate, enhancement, help wanted and [custom labels]
+          }
+          computed
+        }
+      }
+
+
+//      val shouldCount = if (labels.size == 0) true else {
+//        // TODO: this doesn't work for enhancement&&help wanted (counts it, but shouldn't)
+//        val standardCounts = hasLabelOfName(labels, "bug") || hasLabelOfName(labels, "help wanted") || hasLabelOfName(labels, "question")
+//        val helpWantedAndEnhancement = hasLabelOfName(labels, "help wanted") && hasLabelOfName(labels, "enhancement")
+//        val doesNotHaveSomeStandardLabels = !hasSomeStandardGithubLabels(labels)
+//        standardCounts || doesNotHaveSomeStandardLabels
+//      }
+      logger.debug(s"issue ${issue.getNumber} counts = ${shouldCount}, labels = ${labels.map{_.getName}}")
+      shouldCount
+    })
+    count
+  }
+
+  // Issues with these labels ALWAYS count
+  def hasBugOrQuestionLabel(labels: List[GHLabel]): Boolean = {
+    // Future: Eventually we can let custom labels be configured per scraper or per project (OSSMETADATA)
+    hasLabelOfName(labels, "bug") || hasLabelOfName(labels, "question")
+  }
+
+  // Issues with these labels will never count as long as not Bug or Question
+  def hasInvalidOrWontFix(labels: List[GHLabel]): Boolean = {
+    // Future: Eventually we can let custom labels be configured per scraper or per project (OSSMETADATA)
+    hasLabelOfName(labels, "invalid") || hasLabelOfName(labels, "wontfix")
+  }
+
+//  def hasSomeStandardGithubLabels(labels: List[GHLabel]): Boolean = {
+//    hasLabelOfName(labels, "bug") || hasLabelOfName(labels, "help wanted") || hasLabelOfName(labels, "question") ||
+//      hasLabelOfName(labels, "duplicate") || hasLabelOfName(labels, "enhancement") || hasLabelOfName(labels, "invalid") || hasLabelOfName(labels, "wontfix")
+//  }
+
+  def hasLabelOfName(labels: List[GHLabel], name: String): Boolean = {
+    !labels.find(_.getName == name).isEmpty
+  }
+
+  def countLabelForIssues(issues: Array[GHIssue], label: String): Int = {
+    val openCountWithLabelBug: Int = issues.count(issue =>
+      issue.getLabels.size() != 0 &&
+        !issue.getLabels.find(_.getName == label).isEmpty
+    )
+    openCountWithLabelBug
   }
 
   def daysBetween(smaller: Date, bigger: Date): Int = {
@@ -153,11 +294,11 @@ class GithubAccess(val asOfYYYYMMDD: String, val asOfISO: String) {
   }
 
   def getRemainingHourlyRate(): Int = {
-    github.getRateLimit.remaining
+    github.get.getRateLimit.remaining
   }
 
   def getAllRepositoriesForOrg(githubOrg: String): List[GHRepository] = {
-    val org = github.getOrganization(githubOrg)
+    val org = github.get.getOrganization(githubOrg)
     val githubRepos = org.listRepositories(100).asList().toList
     logger.info(s"Found ${githubRepos.size} total repos for ${githubOrg}")
     githubRepos

osstracker-scraper/src/main/scala/com/netflix/oss/tools/osstrackerscraper/GithubScraper.scala

@@ -36,7 +36,7 @@ class GithubScraper(githubOrg: String, cassHost: String, cassPort: Int, esHost:
   def updateElasticSearch(): Boolean = {
     val es = new ElasticSearchAccess(esHost, esPort)
     val cass = new CassandraAccesss(cassHost, cassPort)
-    val github = new GithubAccess(asOfYYYYMMDD, asOfISO)
+    val github = new GithubAccess(asOfYYYYMMDD, asOfISO, true)
 
     try {
       println(Console.RED + s"remaining calls ${github.getRemainingHourlyRate()}" + Console.RESET)
@@ -154,7 +154,7 @@ class GithubScraper(githubOrg: String, cassHost: String, cassPort: Int, esHost:
 
   def updateCassandra(): Boolean = {
     val cass = new CassandraAccesss(cassHost, cassPort)
-    val github = new GithubAccess(asOfYYYYMMDD, asOfISO)
+    val github = new GithubAccess(asOfYYYYMMDD, asOfISO, true)
     val report = StringBuilder.newBuilder
 
     report.append(s"OSSTracker Report for ${asOfYYYYMMDD}\n\n")