Merge pull request #229 from sourcerer-io/develop

feat: limit max commit diff size, fix windows error, add libs

Author: anatolystansler

deploy/Jenkinsfile

@@ -1,35 +1,37 @@
-podTemplate(label: 'build-pod-sourcerer-app',
+// Copyright 2017 Sourcerer, Inc. All Rights Reserved.
+// Author: Maxim Rusak ([email protected])
+
+def label = "sourcerer-app-${UUID.randomUUID().toString()}"
+
+def namespace = 'sandbox'
+if (env.BRANCH_NAME == 'master') {
+  namespace = 'production'
+} else if (env.BRANCH_NAME == 'develop') {
+  namespace = 'staging'
+}
+
+podTemplate(label: label,
     containers: [
         containerTemplate(name: 'jnlp', image: 'gcr.io/sourcerer-1377/jenkins-slave:v4', args: '${computer.jnlpmac} ${computer.name}'),
         containerTemplate(name: 'gradle', image: 'gcr.io/sourcerer-1377/gradle:4.2.0', ttyEnabled: true, command: 'tail -f /dev/null')
     ],
     envVars: [
-      envVar(key: 'CONTAINER_TAG', value: "${env.BRANCH_NAME}.${env.BUILD_NUMBER}")
+      envVar(key: 'NAMESPACE', value: namespace),
+      envVar(key: 'CONTAINER_TAG', value: "${namespace}.${env.BUILD_NUMBER}.${System.currentTimeMillis()}")
     ],
     volumes: [
         hostPathVolume(hostPath: '/var/run/docker.sock', mountPath: '/var/run/docker.sock'),
         hostPathVolume(hostPath: '/usr/bin/docker', mountPath: '/usr/bin/docker')
     ]
 ) {
-  node('build-pod-sourcerer-app') {
-    def namespace = 'sandbox'
-    def plog = 'debug'
-
-    if (env.BRANCH_NAME == 'master') {
-      namespace = 'production'
-      plog = 'info'
-    } else if (env.BRANCH_NAME == 'develop') {
-      namespace = 'staging'
-      plog = 'info'
-    }
-
+  node(label) {
     stage('checkout') {
       checkout scm
     }
 
     stage('build jar and test') {
       container('gradle') {
-        sh("LOG=${plog} NAMESPACE=${namespace} ./do.sh build_jar_inside")
+        sh("./do.sh build_jar_inside")
       }
     }
 
@@ -45,7 +47,7 @@ podTemplate(label: 'build-pod-sourcerer-app',
 
     stage('deploy') {
       println "Deploying to ${namespace} kubernetes namespace"
-      sh("NAMESPACE=${namespace} ./do.sh deploy")
+      sh("./do.sh deploy")
     }
   }
 }

do.sh

@@ -1,35 +1,25 @@
 #!/bin/bash
 
-#-------------------#
-#----- Helpers -----#
-#-------------------#
+# Copyright 2017 Sourcerer, Inc. All Rights Reserved.
+# Author: Maxim Rusak ([email protected])
 
 set -x
 
-usage() {
-    echo "$0 [COMMAND] [ARGUMENTS]"
-    echo "  Commands:"
-    echo "  - build_jar: build jar"
-    echo "  - build_prod_inside: build nginx container"
-    echo "  - run_jar: run jar"
-    echo "  - run_prod: start nginx container"
-}
-
 fn_exists() {
-    type $1 2>/dev/null | grep -q 'is a function'
+  type $1 2>/dev/null | grep -q 'is a function'
 }
 
 COMMAND=$1
 shift
 ARGUMENTS=${@}
 
-TAG="${CONTAINER_TAG:-latest}"
+CONTAINER_TAG="${CONTAINER_TAG:-latest}"
 NAMESPACE="${NAMESPACE:-sandbox}"
 LOG="${LOG:-debug}"
 VOLUME="${BUILD_VOLUME:-$PWD}"
 PROJECT=sourcerer-app
 PORT=3182
-REPO_NAME=gcr.io/sourcerer-1377/$PROJECT:$TAG
+REPO_NAME=gcr.io/sourcerer-1377/$PROJECT:$CONTAINER_TAG
 GRADLE_VERSION=4.2.0
 
 #--------------------#
@@ -40,12 +30,16 @@ GRADLE_VERSION=4.2.0
 build_jar_inside() {
   if [ "$NAMESPACE" == "sandbox" ]; then
     API="https://sandbox.sourcerer/api/commit"
+    LOG="debug"
   elif [ "$NAMESPACE" == "staging" ]; then
     API="https://staging.sourcerer/api/commit"
+    LOG="info"
   elif [ "$NAMESPACE" == "local" ]; then
     API="http://localhost:3181"
+    LOG="debug"
   else
     API="https://sourcerer.io/api/commit"
+    LOG="info"
   fi
   gradle -Penv=$NAMESPACE -Plog=$LOG -Papi=$API build
 }
@@ -88,7 +82,7 @@ push() {
 
 fn_exists $COMMAND
 if [ $? -eq 0 ]; then
-    $COMMAND $ARGUMENTS
+  $COMMAND $ARGUMENTS
 else
-    usage
+  echo "Command not found"
 fi

src/main/kotlin/app/extractors/CppExtractor.kt

@@ -18,7 +18,7 @@ class CppExtractor : ExtractorInterface {
             ExtractorInterface.getMultipleImportsToLibraryMap(LANGUAGE_NAME)
         val importRegex = Regex("""^([^\n]*#include)\s[^\n]*""")
         val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
-        val extractImportRegex = Regex("""#include\s+["<](\w+)[/\w+]*\.\w+[">]""")
+        val extractImportRegex = Regex("""#include\s+["<](\w+)[/\w+]*(\.\w+)?[">]""")
     }
 
     override fun extract(files: List<DiffFile>): List<CommitStats> {
@@ -32,12 +32,19 @@ class CppExtractor : ExtractorInterface {
         fileContent.forEach {
             val res = extractImportRegex.find(it)
             if (res != null) {
-                val lineLib = res.groupValues.last()
+                val lineLib = res.groupValues
+                                 .last { !it.startsWith(".") && it != ""}
                 imports.add(lineLib)
             }
         }
-
         val libraries = imports.map { MULTI_IMPORT_TO_LIB.getOrDefault(it, it) }
+                               .map { import -> when {
+                                   import.startsWith("Q") -> "Qt"
+                                   import.startsWith("Lzma") -> "Lzma"
+                                   import.startsWith("Ogre") -> "Ogre"
+                                   else -> import
+                               }}
+                               .toSet().toList()
         return libraries
     }
 

src/main/kotlin/app/hashers/CommitCrawler.kt

@@ -39,6 +39,7 @@ object CommitCrawler {
     private val LOCAL_HEAD = "HEAD"
     private val REFS = listOf(REMOTE_HEAD, REMOTE_MASTER_BRANCH,
                               LOCAL_MASTER_BRANCH, LOCAL_HEAD)
+    private val MAX_DIFF_SIZE = 600000
 
     fun getDefaultBranchHead(git: Git): ObjectId {
         for (ref in REFS) {
@@ -169,6 +170,11 @@ object CommitCrawler {
             .map { diff ->
                 JgitDiff(diff, df.toFileHeader(diff).toEditList())
             }
+            .filter { diff ->
+                diff.editList.fold(0) { acc, edit ->
+                    acc + edit.lengthA + edit.lengthB
+                } < MAX_DIFF_SIZE
+            }
             subscriber.onNext(JgitPair(commit, diffEdits))
             commit = parentCommit
         }

src/main/kotlin/app/utils/FileHelper.kt

@@ -47,9 +47,8 @@ object FileHelper {
     }
 
     fun getJarPath(): Path {
-        val fullPathString = URLDecoder.decode(FileHelper::class.java
-            .protectionDomain.codeSource.location.toURI().path, "UTF-8")
-        val fullPath = Paths.get(fullPathString)
+        val fullPathURI = FileHelper::class.java.protectionDomain.codeSource.location.toURI()
+        val fullPath = Paths.get(fullPathURI)
         val root = fullPath.root
         // Removing jar filename.
         return root.resolve(fullPath.subpath(0, fullPath.nameCount - 1))

src/main/resources/data/libraries/java_libraries.txt

@@ -45,7 +45,7 @@ com.airbnb.android.react.lottie
 zipkin
 com.facebook.presto
 dagger
-junit
+org.junit
 com.lmax.disruptor
 com.jakewharton.rxbinding2
 org.physical_web

src/test/kotlin/test/tests/extractors/ExtractorTest.kt

@@ -259,4 +259,12 @@ class ExtractorTest : Spek({
         }
     }
 
+    given("Qt import in cpp file") {
+        it("extracts library name") {
+            val lib = "Qt"
+            val line = "#include <QFileDialog>"
+            assertExtractsImport(lib, line, CppExtractor())
+        }
+    }
+
 })